Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,33 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/finite_state_turn_taking_raux_2009.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/focal_loss_lin_2017.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/01-kosmala-crible-2022-dual-status-filled-pauses.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/02-ekstedt-skantze-2022-prosody-turn-taking-vap.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/03-castillo-lopez-2025-survey-turn-taking-modeling.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/04-cenoz-2000-pauses-hesitation-L2-production.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/05-kosmala-2023-filled-pauses-pragmatic-markers-gaze-gesture.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/06-wu-2023-disfluencies-french-prosodic-params-filled-pauses.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/07-ekstedt-2023-turn-taking-cues-speech-synthesis.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/08-inoue-2024-realtime-turn-taking-vap.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/09-inoue-2024-multilingual-turn-taking-vap.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/10-hutin-2024-filled-pauses-conversational-convergence.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/11-beatty-martinez-2020-codeswitching-bilingual-toolkit.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/12-lingref-code-switching-bilinguals.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/13-christodoulides-avanzi-2014-DisMo-disfluency-french.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/14-jouvet-2019-speech-processing-prosody.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/15-filler-particles-cross-linguistic-heritage-2024.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/16-hal-peters-2017-L2-fluency-french.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/17-christodoulides-2015-automatic-disfluency-detection-french.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/hesitation-l2-french/19-hal-multimodal-self-other-repair-L2.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/language-learning-turn-taking/conversar_mixed_reality_l2.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/language-learning-turn-taking/hesitation_tagging_l2_whisper_lora.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/language-learning-turn-taking/iwsds2025_survey_turn_taking.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/language-learning-turn-taking/multilingual_vap_2024.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/language-learning-turn-taking/speak_improve_corpus_2025.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/soda_dialog_distillation_2023.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/speculative_etd_2025.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/turn_taking_review_skantze_2021.pdf filter=lfs diff=lfs merge=lfs -text
+03-finetune-pipecat-pt/references/papers/vap_turn_taking_ekstedt_2024.pdf filter=lfs diff=lfs merge=lfs -text
+previous-experiments/01-benchmarks/report/figures/radar_chart.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,11 @@
+data/
+checkpoints/
+__pycache__/
+*.pyc
+.deploy_state.json
+benchmark.log
+*.egg-info/
+hf_cache/
+results/
+*.pt
+*.onnx

03-finetune-pipecat-pt/01_download_pipecat.py

@@ -0,0 +1,176 @@
+"""Download Pipecat Smart Turn v3 dataset + model architecture.
+
+The Pipecat model is only published as ONNX (no PyTorch weights).
+So we initialize from openai/whisper-tiny and train using their dataset
+which already includes Portuguese samples.
+
+This script:
+1. Downloads Pipecat's training dataset (270K samples, 23 langs)
+2. Filters Portuguese samples
+3. Downloads the ONNX model for reference/benchmarking
+4. Saves Portuguese subset locally
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+DATA_DIR = Path(__file__).parent / "data"
+CACHE_DIR = Path("/workspace/hf_cache") if Path("/workspace").exists() else DATA_DIR / "hf_cache"
+
+
+def download_pipecat_dataset(max_pt_samples: int = 5000) -> Path:
+    """Download Pipecat v3.2 training data and extract Portuguese samples."""
+    from datasets import load_dataset
+
+    DATA_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Download full dataset (streaming to avoid 37GB download)
+    log.info("Loading Pipecat Smart Turn v3.2 training data (streaming)...")
+    ds = load_dataset(
+        "pipecat-ai/smart-turn-data-v3.2-train",
+        split="train",
+        streaming=True,
+        cache_dir=str(CACHE_DIR),
+    )
+
+    # Filter Portuguese samples
+    pt_samples = []
+    other_count = 0
+    for row in ds:
+        lang = row.get("language", "")
+        if lang == "por":
+            pt_samples.append({
+                "id": row["id"],
+                "language": lang,
+                "endpoint_bool": row["endpoint_bool"],
+                "midfiller": row.get("midfiller", False),
+                "endfiller": row.get("endfiller", False),
+                "synthetic": row.get("synthetic", True),
+                "dataset": row.get("dataset", ""),
+                "spoken_text": row.get("spoken_text", ""),
+                "audio": row["audio"],
+            })
+            if len(pt_samples) % 100 == 0:
+                log.info("  Found %d Portuguese samples (scanned %d others)",
+                         len(pt_samples), other_count)
+            if len(pt_samples) >= max_pt_samples:
+                break
+        else:
+            other_count += 1
+
+    log.info("Total: %d Portuguese samples found (scanned %d other languages)",
+             len(pt_samples), other_count)
+
+    # Save metadata (without audio)
+    meta_path = DATA_DIR / "pipecat_pt_metadata.json"
+    meta = [{k: v for k, v in s.items() if k != "audio"} for s in pt_samples]
+    with open(meta_path, "w") as f:
+        json.dump(meta, f, indent=2, ensure_ascii=False)
+    log.info("Metadata saved to %s", meta_path)
+
+    # Save audio files
+    audio_dir = DATA_DIR / "pipecat_pt_audio"
+    audio_dir.mkdir(exist_ok=True)
+
+    import soundfile as sf
+    import numpy as np
+
+    complete = 0
+    incomplete = 0
+    for s in pt_samples:
+        audio_data = s["audio"]
+        audio = np.array(audio_data["array"], dtype=np.float32)
+        sr = audio_data["sampling_rate"]
+
+        label = "complete" if s["endpoint_bool"] else "incomplete"
+        if s["endpoint_bool"]:
+            complete += 1
+        else:
+            incomplete += 1
+
+        out_path = audio_dir / f"{s['id']}_{label}.wav"
+        sf.write(str(out_path), audio, sr)
+
+    log.info("Audio saved: %d complete, %d incomplete → %s",
+             complete, incomplete, audio_dir)
+
+    return audio_dir
+
+
+def download_pipecat_test_data(max_pt_samples: int = 2000) -> Path:
+    """Download Pipecat test data for evaluation."""
+    from datasets import load_dataset
+
+    log.info("Loading Pipecat Smart Turn v3.2 test data (streaming)...")
+    ds = load_dataset(
+        "pipecat-ai/smart-turn-data-v3.2-test",
+        split="train",
+        streaming=True,
+        cache_dir=str(CACHE_DIR),
+    )
+
+    pt_samples = []
+    for row in ds:
+        if row.get("language", "") == "por":
+            pt_samples.append(row)
+            if len(pt_samples) >= max_pt_samples:
+                break
+
+    log.info("Found %d Portuguese test samples", len(pt_samples))
+
+    # Save
+    test_dir = DATA_DIR / "pipecat_pt_test"
+    test_dir.mkdir(exist_ok=True)
+
+    import soundfile as sf
+    import numpy as np
+
+    for s in pt_samples:
+        audio = np.array(s["audio"]["array"], dtype=np.float32)
+        sr = s["audio"]["sampling_rate"]
+        label = "complete" if s["endpoint_bool"] else "incomplete"
+        sf.write(str(test_dir / f"{s['id']}_{label}.wav"), audio, sr)
+
+    log.info("Test audio saved → %s", test_dir)
+    return test_dir
+
+
+def download_onnx_model() -> Path:
+    """Download Pipecat ONNX model for benchmarking."""
+    from huggingface_hub import hf_hub_download
+
+    model_dir = DATA_DIR / "pipecat_model"
+    model_dir.mkdir(exist_ok=True)
+
+    for fname in ["smart-turn-v3.2-cpu.onnx", "smart-turn-v3.2-gpu.onnx"]:
+        path = hf_hub_download(
+            "pipecat-ai/smart-turn-v3",
+            fname,
+            local_dir=str(model_dir),
+        )
+        log.info("Downloaded %s → %s", fname, path)
+
+    return model_dir
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s [%(name)s] %(message)s",
+    )
+
+    log.info("=== Step 1: Download Pipecat Portuguese training data ===")
+    download_pipecat_dataset(max_pt_samples=5000)
+
+    log.info("\n=== Step 2: Download Pipecat Portuguese test data ===")
+    download_pipecat_test_data(max_pt_samples=2000)
+
+    log.info("\n=== Step 3: Download ONNX model for benchmarking ===")
+    download_onnx_model()
+
+    log.info("\nDone!")

03-finetune-pipecat-pt/02_generate_labels.py

@@ -0,0 +1,474 @@
+"""Generate high-quality turn-taking labels using Claude API.
+
+Based on the Pipecat v3.1 pipeline:
+1. Load Portuguese transcripts from CORAA MuPe + NURC-SP
+2. Claude filters bad/ambiguous sentences (Pipecat: Gemini removed 50-80%)
+3. Claude classifies: COMPLETE vs INCOMPLETE (semantic, not punctuation-based)
+4. Claude inserts Brazilian Portuguese fillers
+5. Claude generates French-accented Portuguese variants
+
+Uses Claude Haiku for cost efficiency (~$0.001 per 20 sentences).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+DATA_DIR = Path(__file__).parent / "data"
+CACHE_DIR = Path("/workspace/hf_cache") if Path("/workspace").exists() else DATA_DIR / "hf_cache"
+
+# Brazilian Portuguese fillers (from research: Claude + GPT generate these for Pipecat)
+PT_BR_FILLERS = [
+    "hum", "eh", "ah", "tipo", "ne", "entao", "assim", "quer dizer",
+    "como e que eu falo", "deixa eu pensar", "olha", "bom", "pois e",
+    "sabe", "veja bem", "na verdade", "digamos", "e...",
+]
+
+# French speaker fillers when speaking Portuguese
+FR_PT_FILLERS = [
+    "euh", "alors", "comment dire", "como se diz", "enfin",
+    "c'est-a-dire", "voila", "bon", "donc",
+]
+
+
+@dataclass
+class LabeledSentence:
+    text: str
+    label: str  # "complete" or "incomplete"
+    confidence: float  # 0-1
+    source: str  # "coraa", "mupe", "claude_generated"
+    has_filler: bool = False
+    filler_type: str = ""  # "pt_br" or "fr_pt"
+    original_text: str = ""
+
+
+def load_coraa_transcripts(max_samples: int = 10000) -> list[dict]:
+    """Load transcripts from CORAA MuPe (365h interviews, diarization kappa 0.947)."""
+    from datasets import load_dataset
+
+    log.info("Loading CORAA MuPe transcripts (streaming)...")
+    ds = load_dataset(
+        "nilc-nlp/CORAA-MUPE-ASR",
+        split="train",
+        streaming=True,
+        cache_dir=str(CACHE_DIR),
+    )
+
+    transcripts = []
+    for i, row in enumerate(ds):
+        text = str(row.get("text", row.get("normalized_text", "")))
+        if not text or len(text.split()) < 3:
+            continue
+
+        transcripts.append({
+            "text": text,
+            "speaker_type": row.get("speaker_type", ""),
+            "speaker_code": str(row.get("speaker_code", f"mupe_{i}")),
+            "source": "mupe",
+        })
+
+        if len(transcripts) >= max_samples:
+            break
+
+        if i % 5000 == 0 and i > 0:
+            log.info("  Scanned %d rows, collected %d transcripts", i, len(transcripts))
+
+    log.info("CORAA MuPe: %d transcripts loaded", len(transcripts))
+    return transcripts
+
+
+def load_nurc_transcripts(max_samples: int = 10000) -> list[dict]:
+    """Load transcripts from CORAA NURC-SP (239h dialogues, speaker IDs)."""
+    from datasets import load_dataset
+
+    log.info("Loading CORAA NURC-SP transcripts (streaming)...")
+    try:
+        ds = load_dataset(
+            "nilc-nlp/CORAA-NURC-SP-Audio-Corpus",
+            split="train",
+            streaming=True,
+            cache_dir=str(CACHE_DIR),
+        )
+    except Exception as e:
+        log.warning("Failed to load NURC-SP: %s", e)
+        return []
+
+    transcripts = []
+    for i, row in enumerate(ds):
+        text = str(row.get("text", row.get("sentence", "")))
+        if not text or len(text.split()) < 3:
+            continue
+
+        transcripts.append({
+            "text": text,
+            "speaker_type": row.get("speech_genre", ""),
+            "speaker_code": str(row.get("speaker_id", f"nurc_{i}")),
+            "source": "nurc_sp",
+        })
+
+        if len(transcripts) >= max_samples:
+            break
+
+    log.info("NURC-SP: %d transcripts loaded", len(transcripts))
+    return transcripts
+
+
+def classify_with_claude(
+    sentences: list[str],
+    batch_size: int = 20,
+    model: str = "claude-haiku-4-5-20251001",
+) -> list[dict]:
+    """Use Claude to classify sentences as complete/incomplete.
+
+    Based on Pipecat's approach: Gemini 2.5 Flash filtered 50-80% of sentences.
+    We use Claude Haiku for cost efficiency.
+
+    Returns list of {text, label, confidence, keep}.
+    """
+    import anthropic
+
+    api_key = os.environ.get("ANTHROPIC_API_KEY")
+    if not api_key:
+        log.warning("ANTHROPIC_API_KEY not set — using rule-based fallback")
+        return _rule_based_classify(sentences)
+
+    client = anthropic.Anthropic(api_key=api_key)
+    results = []
+
+    for batch_start in range(0, len(sentences), batch_size):
+        batch = sentences[batch_start:batch_start + batch_size]
+        numbered = "\n".join(f"{i+1}. {s}" for i, s in enumerate(batch))
+
+        prompt = f"""Voce e um anotador de turn-taking para portugues brasileiro.
+Para cada frase abaixo, classifique:
+- COMPLETO: o falante terminou o pensamento (pode comecar a traduzir)
+- INCOMPLETO: o falante vai continuar falando (nao traduzir ainda)
+- RUIM: frase com erro, ambigua, ou inutilizavel (descartar)
+
+Responda APENAS em JSON, sem explicacao:
+[{{"n": 1, "label": "COMPLETO", "confidence": 0.95}}, ...]
+
+Frases:
+{numbered}"""
+
+        try:
+            response = client.messages.create(
+                model=model,
+                max_tokens=2000,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            text = response.content[0].text
+
+            # Parse JSON from response
+            json_match = re.search(r'\[.*\]', text, re.DOTALL)
+            if json_match:
+                batch_results = json.loads(json_match.group())
+                for item in batch_results:
+                    idx = item["n"] - 1
+                    if 0 <= idx < len(batch):
+                        results.append({
+                            "text": batch[idx],
+                            "label": item["label"].lower(),
+                            "confidence": item.get("confidence", 0.5),
+                            "keep": item["label"].upper() != "RUIM",
+                        })
+            else:
+                log.warning("No JSON in Claude response for batch %d", batch_start)
+                results.extend(_rule_based_classify(batch))
+
+        except Exception as e:
+            log.warning("Claude API error at batch %d: %s — falling back to rules", batch_start, e)
+            results.extend(_rule_based_classify(batch))
+
+        # Rate limiting
+        if batch_start % 100 == 0 and batch_start > 0:
+            log.info("  Classified %d/%d sentences", batch_start, len(sentences))
+            time.sleep(0.5)
+
+    return results
+
+
+def _rule_based_classify(sentences: list[str]) -> list[dict]:
+    """Fallback rule-based classification (less accurate than Claude)."""
+    results = []
+    for s in sentences:
+        text = s.strip()
+        if not text or len(text) < 5:
+            results.append({"text": text, "label": "ruim", "confidence": 0.5, "keep": False})
+            continue
+
+        if re.search(r'[.!?]+\s*$', text):
+            results.append({"text": text, "label": "completo", "confidence": 0.7, "keep": True})
+        elif re.search(r'[,;:\-]\s*$', text):
+            results.append({"text": text, "label": "incompleto", "confidence": 0.6, "keep": True})
+        elif re.search(r'\b(e|que|mas|porque|quando|se|como|pra|para)\s*$', text, re.I):
+            results.append({"text": text, "label": "incompleto", "confidence": 0.8, "keep": True})
+        else:
+            results.append({"text": text, "label": "completo", "confidence": 0.5, "keep": True})
+
+    return results
+
+
+def insert_fillers_with_claude(
+    sentences: list[str],
+    filler_type: str = "pt_br",
+    model: str = "claude-haiku-4-5-20251001",
+) -> list[dict]:
+    """Use Claude to insert fillers at natural points.
+
+    Based on Pipecat: Gemini Flash inserted fillers at natural break points.
+
+    filler_type: "pt_br" for native Brazilian, "fr_pt" for French-accented
+    """
+    import anthropic
+
+    api_key = os.environ.get("ANTHROPIC_API_KEY")
+    if not api_key:
+        log.warning("ANTHROPIC_API_KEY not set — using simple filler insertion")
+        return _simple_filler_insert(sentences, filler_type)
+
+    client = anthropic.Anthropic(api_key=api_key)
+    fillers = PT_BR_FILLERS if filler_type == "pt_br" else FR_PT_FILLERS
+
+    filler_list = ", ".join(f'"{f}"' for f in fillers)
+
+    if filler_type == "fr_pt":
+        context = """O falante e um frances de nivel B1 falando portugues.
+Ele hesita ao conjugar verbos, busca palavras, e as vezes usa fillers em frances.
+Insira hesitacoes naturais como: """ + filler_list
+    else:
+        context = """O falante e brasileiro nativo.
+Insira fillers naturais do portugues brasileiro como: """ + filler_list
+
+    results = []
+    batch_size = 15
+
+    for batch_start in range(0, len(sentences), batch_size):
+        batch = sentences[batch_start:batch_start + batch_size]
+        numbered = "\n".join(f"{i+1}. {s}" for i, s in enumerate(batch))
+
+        prompt = f"""{context}
+
+Para cada frase, crie uma versao com 1-2 fillers inseridos em pontos naturais.
+A frase deve parecer INCOMPLETA (como se o falante fosse continuar).
+
+Responda APENAS em JSON:
+[{{"n": 1, "original": "frase original", "with_filler": "frase com filler"}}]
+
+Frases:
+{numbered}"""
+
+        try:
+            response = client.messages.create(
+                model=model,
+                max_tokens=3000,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            text = response.content[0].text
+            json_match = re.search(r'\[.*\]', text, re.DOTALL)
+            if json_match:
+                batch_results = json.loads(json_match.group())
+                for item in batch_results:
+                    results.append({
+                        "original": item.get("original", ""),
+                        "with_filler": item.get("with_filler", ""),
+                        "filler_type": filler_type,
+                    })
+
+        except Exception as e:
+            log.warning("Claude filler API error: %s", e)
+            results.extend(_simple_filler_insert(batch, filler_type))
+
+        time.sleep(0.3)
+
+    return results
+
+
+def _simple_filler_insert(sentences: list[str], filler_type: str) -> list[dict]:
+    """Simple rule-based filler insertion fallback."""
+    import random
+
+    fillers = PT_BR_FILLERS if filler_type == "pt_br" else FR_PT_FILLERS
+    results = []
+
+    for s in sentences:
+        words = s.split()
+        if len(words) < 4:
+            results.append({"original": s, "with_filler": s, "filler_type": filler_type})
+            continue
+
+        # Insert filler at ~40% of sentence
+        pos = max(1, len(words) * 2 // 5)
+        filler = random.choice(fillers)
+        words.insert(pos, f"{filler}...")
+        results.append({
+            "original": s,
+            "with_filler": " ".join(words),
+            "filler_type": filler_type,
+        })
+
+    return results
+
+
+def generate_french_portuguese_sentences(
+    n_sentences: int = 500,
+    model: str = "claude-haiku-4-5-20251001",
+) -> list[dict]:
+    """Generate sentences typical of French speakers learning Portuguese.
+
+    Based on Pipecat method: use LLM to generate diverse training text.
+    """
+    import anthropic
+
+    api_key = os.environ.get("ANTHROPIC_API_KEY")
+    if not api_key:
+        log.warning("ANTHROPIC_API_KEY not set — cannot generate French-PT sentences")
+        return []
+
+    client = anthropic.Anthropic(api_key=api_key)
+    results = []
+
+    contexts = [
+        "reuniao de trabalho",
+        "conversa informal com amigos",
+        "apresentacao de projeto",
+        "negociacao comercial",
+        "aula de portugues",
+        "pedindo informacoes na rua",
+        "restaurante pedindo comida",
+        "entrevista de emprego",
+        "ligacao telefonica profissional",
+        "discussao tecnica sobre software",
+    ]
+
+    for ctx in contexts:
+        prompt = f"""Gere {n_sentences // len(contexts)} frases que um FRANCES de nivel B1-B2 diria em portugues durante: {ctx}
+
+Inclua variedade:
+- Frases COMPLETAS (pensamento terminado, pode traduzir)
+- Frases INCOMPLETAS (vai continuar falando, nao traduzir)
+- Com hesitacoes tipicas (euh, alors, como se diz, tipo)
+- Com erros de conjugacao comuns de franceses
+- Com code-switching involuntario (palavra em frances no meio)
+
+Responda em JSON:
+[{{"text": "frase", "label": "completo" ou "incompleto", "notes": "o que torna dificil classificar"}}]"""
+
+        try:
+            response = client.messages.create(
+                model=model,
+                max_tokens=4000,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            text = response.content[0].text
+            json_match = re.search(r'\[.*\]', text, re.DOTALL)
+            if json_match:
+                batch = json.loads(json_match.group())
+                for item in batch:
+                    results.append({
+                        "text": item["text"],
+                        "label": item["label"],
+                        "source": "claude_fr_pt",
+                        "context": ctx,
+                        "notes": item.get("notes", ""),
+                    })
+                log.info("  Generated %d sentences for context: %s", len(batch), ctx)
+
+        except Exception as e:
+            log.warning("Error generating for %s: %s", ctx, e)
+
+        time.sleep(1)
+
+    log.info("Total French-PT sentences generated: %d", len(results))
+    return results
+
+
+def run_full_pipeline(
+    max_transcripts: int = 5000,
+    max_fr_sentences: int = 500,
+) -> Path:
+    """Run the full label generation pipeline."""
+    output_dir = DATA_DIR / "claude_labeled"
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Step 1: Load transcripts
+    log.info("=== Step 1: Loading Portuguese transcripts ===")
+    mupe = load_coraa_transcripts(max_samples=max_transcripts)
+    nurc = load_nurc_transcripts(max_samples=max_transcripts)
+    all_transcripts = mupe + nurc
+    log.info("Total transcripts: %d (MuPe=%d, NURC=%d)", len(all_transcripts), len(mupe), len(nurc))
+
+    # Step 2: Claude classifies
+    log.info("\n=== Step 2: Claude classifies complete/incomplete ===")
+    sentences = [t["text"] for t in all_transcripts]
+    classified = classify_with_claude(sentences)
+
+    # Filter bad sentences (Pipecat removed 50-80%)
+    kept = [c for c in classified if c["keep"]]
+    removed = len(classified) - len(kept)
+    log.info("Classified: %d total, %d kept, %d removed (%.0f%%)",
+             len(classified), len(kept), removed, 100 * removed / max(len(classified), 1))
+
+    complete = [c for c in kept if c["label"] == "completo"]
+    incomplete = [c for c in kept if c["label"] == "incompleto"]
+    log.info("  Complete: %d, Incomplete: %d", len(complete), len(incomplete))
+
+    # Save classified
+    with open(output_dir / "classified_pt.json", "w") as f:
+        json.dump(kept, f, indent=2, ensure_ascii=False)
+
+    # Step 3: Insert fillers (creates INCOMPLETE variants)
+    log.info("\n=== Step 3: Insert fillers (PT-BR + FR-PT) ===")
+    complete_texts = [c["text"] for c in complete[:1000]]
+
+    pt_fillers = insert_fillers_with_claude(complete_texts, filler_type="pt_br")
+    fr_fillers = insert_fillers_with_claude(complete_texts[:500], filler_type="fr_pt")
+
+    with open(output_dir / "fillers_pt_br.json", "w") as f:
+        json.dump(pt_fillers, f, indent=2, ensure_ascii=False)
+    with open(output_dir / "fillers_fr_pt.json", "w") as f:
+        json.dump(fr_fillers, f, indent=2, ensure_ascii=False)
+
+    log.info("Fillers: %d PT-BR, %d FR-PT", len(pt_fillers), len(fr_fillers))
+
+    # Step 4: Generate French-Portuguese sentences
+    log.info("\n=== Step 4: Generate French-Portuguese sentences ===")
+    fr_sentences = generate_french_portuguese_sentences(n_sentences=max_fr_sentences)
+    with open(output_dir / "french_portuguese.json", "w") as f:
+        json.dump(fr_sentences, f, indent=2, ensure_ascii=False)
+
+    # Summary
+    summary = {
+        "total_transcripts": len(all_transcripts),
+        "classified_kept": len(kept),
+        "classified_removed": removed,
+        "complete": len(complete),
+        "incomplete": len(incomplete),
+        "fillers_pt_br": len(pt_fillers),
+        "fillers_fr_pt": len(fr_fillers),
+        "french_portuguese": len(fr_sentences),
+    }
+    with open(output_dir / "summary.json", "w") as f:
+        json.dump(summary, f, indent=2)
+
+    log.info("\n=== Summary ===")
+    for k, v in summary.items():
+        log.info("  %s: %d", k, v)
+
+    return output_dir
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s [%(name)s] %(message)s",
+    )
+    run_full_pipeline(max_transcripts=5000, max_fr_sentences=500)

03-finetune-pipecat-pt/03_generate_audio.py

@@ -0,0 +1,697 @@
+"""Generate TTS audio for the turn-taking dataset.
+
+Based on the Pipecat v3.1 pipeline:
+1. Native PT-BR voices via Kokoro (pf_dora, pm_alex, pm_santa)
+2. French-accented Portuguese via XTTS v2 voice cloning
+3. Speed/pitch variation + noise augmentation
+4. Hesitation pause injection (1.5-3s silence after fillers) — SpeculativeETD V3
+5. Short utterance dataset ("sim", "nao", "ok") — Pipecat v3.2 (-40% errors)
+
+Pipecat used Google Chirp3 TTS; we use Kokoro (open-source, runs locally)
++ XTTS v2 (voice cloning for accent simulation).
+
+Run locally or on Modal GPU:
+    python 03_generate_audio.py
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import random
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+
+log = logging.getLogger(__name__)
+
+DATA_DIR = Path(__file__).parent / "data"
+SAMPLE_RATE = 16000  # Pipecat model expects 16kHz
+
+
+@dataclass
+class AudioSample:
+    """A single audio sample for the dataset."""
+    audio: np.ndarray      # float32, 16kHz
+    text: str
+    label: str             # "complete" or "incomplete"
+    voice: str             # e.g. "pf_dora", "fr_clone_1"
+    accent: str            # "native_pt_br" or "french_pt"
+    source: str            # "kokoro", "xtts", "pipecat"
+    speed: float = 1.0
+
+
+# ---------------------------------------------------------------------------
+# Kokoro TTS — native PT-BR voices
+# ---------------------------------------------------------------------------
+
+def generate_kokoro_audio(
+    sentences: list[dict],
+    voices: list[str] | None = None,
+    speed_range: tuple[float, float] = (0.85, 1.15),
+) -> list[AudioSample]:
+    """Generate audio using Kokoro TTS (PT-BR voices).
+
+    sentences: list of {"text": str, "label": "complete"|"incomplete"}
+    voices: Kokoro voice IDs (default: PT-BR voices)
+    """
+    try:
+        from kokoro import KPipeline
+    except ImportError:
+        log.error("kokoro not installed — pip install kokoro")
+        return []
+
+    if voices is None:
+        voices = ["pf_dora", "pm_alex", "pm_santa"]
+
+    log.info("Initializing Kokoro pipeline (lang=pt)...")
+    pipeline = KPipeline(lang_code="p")  # Portuguese
+
+    samples = []
+    errors = 0
+
+    for i, sent in enumerate(sentences):
+        text = sent["text"]
+        label = sent["label"]
+        voice = random.choice(voices)
+        speed = random.uniform(*speed_range)
+
+        try:
+            # Generate audio
+            generator = pipeline(text, voice=voice, speed=speed)
+            audio_chunks = []
+            for _, _, chunk in generator:
+                audio_chunks.append(chunk.numpy() if hasattr(chunk, 'numpy') else np.array(chunk))
+
+            if not audio_chunks:
+                errors += 1
+                continue
+
+            audio = np.concatenate(audio_chunks).astype(np.float32)
+
+            # Resample to 16kHz if needed (Kokoro outputs 24kHz)
+            if hasattr(pipeline, 'sample_rate') and pipeline.sample_rate != SAMPLE_RATE:
+                import torchaudio
+                import torch
+                tensor = torch.from_numpy(audio).float().unsqueeze(0)
+                audio = torchaudio.functional.resample(
+                    tensor, pipeline.sample_rate, SAMPLE_RATE
+                ).squeeze().numpy()
+
+            # Normalize
+            peak = np.max(np.abs(audio))
+            if peak > 0:
+                audio = audio / peak * 0.9
+
+            samples.append(AudioSample(
+                audio=audio,
+                text=text,
+                label=label,
+                voice=voice,
+                accent="native_pt_br",
+                source="kokoro",
+                speed=speed,
+            ))
+
+        except Exception as e:
+            errors += 1
+            if errors <= 5:
+                log.warning("Kokoro error on sentence %d: %s", i, e)
+
+        if (i + 1) % 100 == 0:
+            log.info("  Kokoro: %d/%d generated (%d errors)", len(samples), i + 1, errors)
+
+    log.info("Kokoro: %d samples generated (%d errors)", len(samples), errors)
+    return samples
+
+
+# ---------------------------------------------------------------------------
+# XTTS v2 — French-accented Portuguese via voice cloning
+# ---------------------------------------------------------------------------
+
+def generate_xtts_audio(
+    sentences: list[dict],
+    reference_audio_dir: Path | None = None,
+    speed_range: tuple[float, float] = (0.9, 1.1),
+) -> list[AudioSample]:
+    """Generate audio with French accent using XTTS v2 voice cloning.
+
+    Uses short French speech samples as reference to clone the accent,
+    then synthesizes Portuguese text with that voice.
+
+    sentences: list of {"text": str, "label": "complete"|"incomplete"}
+    reference_audio_dir: directory with .wav files of French speakers
+    """
+    try:
+        from TTS.api import TTS
+    except ImportError:
+        log.error("TTS (coqui) not installed — pip install TTS")
+        return []
+
+    log.info("Initializing XTTS v2...")
+    tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2")
+
+    # Find reference audio files (French speakers)
+    ref_files = []
+    if reference_audio_dir and reference_audio_dir.exists():
+        ref_files = list(reference_audio_dir.glob("*.wav"))
+
+    if not ref_files:
+        log.warning("No French reference audio found in %s — using default voice", reference_audio_dir)
+        # Fall back to generating without cloning (still works, but no accent)
+        return _generate_xtts_no_clone(tts, sentences, speed_range)
+
+    log.info("Found %d French reference voices", len(ref_files))
+
+    samples = []
+    errors = 0
+
+    for i, sent in enumerate(sentences):
+        text = sent["text"]
+        label = sent["label"]
+        ref = random.choice(ref_files)
+
+        try:
+            audio = tts.tts(
+                text=text,
+                speaker_wav=str(ref),
+                language="pt",
+            )
+            audio = np.array(audio, dtype=np.float32)
+
+            # Resample to 16kHz if needed
+            if hasattr(tts, 'synthesizer') and tts.synthesizer.output_sample_rate != SAMPLE_RATE:
+                import torchaudio
+                import torch
+                tensor = torch.from_numpy(audio).float().unsqueeze(0)
+                audio = torchaudio.functional.resample(
+                    tensor, tts.synthesizer.output_sample_rate, SAMPLE_RATE
+                ).squeeze().numpy()
+
+            # Normalize
+            peak = np.max(np.abs(audio))
+            if peak > 0:
+                audio = audio / peak * 0.9
+
+            # Random speed variation
+            speed = random.uniform(*speed_range)
+            if abs(speed - 1.0) > 0.05:
+                indices = np.arange(0, len(audio), speed).astype(int)
+                indices = indices[indices < len(audio)]
+                audio = audio[indices]
+
+            samples.append(AudioSample(
+                audio=audio,
+                text=text,
+                label=label,
+                voice=f"fr_clone_{ref.stem}",
+                accent="french_pt",
+                source="xtts",
+                speed=speed,
+            ))
+
+        except Exception as e:
+            errors += 1
+            if errors <= 5:
+                log.warning("XTTS error on sentence %d: %s", i, e)
+
+        if (i + 1) % 50 == 0:
+            log.info("  XTTS: %d/%d generated (%d errors)", len(samples), i + 1, errors)
+
+    log.info("XTTS: %d samples generated (%d errors)", len(samples), errors)
+    return samples
+
+
+def _generate_xtts_no_clone(
+    tts,
+    sentences: list[dict],
+    speed_range: tuple[float, float],
+) -> list[AudioSample]:
+    """XTTS generation without voice cloning (fallback)."""
+    samples = []
+    errors = 0
+
+    for i, sent in enumerate(sentences):
+        try:
+            audio = tts.tts(text=sent["text"], language="pt")
+            audio = np.array(audio, dtype=np.float32)
+
+            peak = np.max(np.abs(audio))
+            if peak > 0:
+                audio = audio / peak * 0.9
+
+            samples.append(AudioSample(
+                audio=audio,
+                text=sent["text"],
+                label=sent["label"],
+                voice="xtts_default",
+                accent="french_pt",
+                source="xtts",
+                speed=1.0,
+            ))
+        except Exception as e:
+            errors += 1
+            if errors <= 3:
+                log.warning("XTTS fallback error: %s", e)
+
+    log.info("XTTS (no clone): %d samples (%d errors)", len(samples), errors)
+    return samples
+
+
+# ---------------------------------------------------------------------------
+# Hesitation pause injection (SpeculativeETD V3 — best data variant)
+# ---------------------------------------------------------------------------
+
+def inject_hesitation_pause(
+    audio: np.ndarray,
+    pause_duration_range: tuple[float, float] = (1.5, 3.0),
+    position: str = "end",
+) -> np.ndarray:
+    """Inject a realistic hesitation pause into audio.
+
+    SpeculativeETD V3 showed that inserting 1.5-3.0s pauses after fillers
+    was the most effective data variant for training ETD models.
+
+    For French speakers learning Portuguese, these long pauses happen when:
+    - Searching for a word ("Eu preciso de... [2s pause] ...uma tesoura")
+    - Thinking about conjugation ("Ontem eu... [2s pause] ...fui ao mercado")
+    - Code-switching hesitation ("Eu gosto de... [1.5s] ...euh... praia")
+
+    position: "end" = pause at end (simulates mid-utterance stop)
+              "mid" = pause in the middle (simulates hesitation)
+    """
+    pause_s = random.uniform(*pause_duration_range)
+    pause_samples = int(pause_s * SAMPLE_RATE)
+
+    # Add very low-level noise to the pause (not pure silence — more realistic)
+    pause = np.random.randn(pause_samples).astype(np.float32) * 0.001
+
+    if position == "end":
+        # Pause at end: speaker stops mid-sentence (INCOMPLETE)
+        return np.concatenate([audio, pause])
+    else:
+        # Pause in middle: split audio and insert pause
+        if len(audio) < SAMPLE_RATE:  # too short to split
+            return np.concatenate([audio, pause])
+        split_point = random.randint(len(audio) // 3, 2 * len(audio) // 3)
+        return np.concatenate([audio[:split_point], pause, audio[split_point:]])
+
+
+def create_hesitation_variants(
+    samples: list[AudioSample],
+    fraction: float = 0.3,
+) -> list[AudioSample]:
+    """Create hesitation-pause variants from existing INCOMPLETE samples.
+
+    Takes a fraction of incomplete samples and adds 1.5-3s pauses,
+    simulating French speakers hesitating in Portuguese.
+    """
+    incomplete = [s for s in samples if s.label == "incomplete"]
+    n_variants = int(len(incomplete) * fraction)
+
+    variants = []
+    for s in random.sample(incomplete, min(n_variants, len(incomplete))):
+        # Variant 1: long pause at end (speaker stopped to think)
+        audio_end = inject_hesitation_pause(s.audio, position="end")
+        variants.append(AudioSample(
+            audio=audio_end,
+            text=s.text,
+            label="incomplete",  # still incomplete — speaker will continue
+            voice=s.voice,
+            accent=s.accent,
+            source=f"{s.source}_hesitation_end",
+            speed=s.speed,
+        ))
+
+        # Variant 2: pause in the middle (thinking mid-sentence)
+        if random.random() < 0.5:
+            audio_mid = inject_hesitation_pause(s.audio, position="mid")
+            variants.append(AudioSample(
+                audio=audio_mid,
+                text=s.text,
+                label="incomplete",
+                voice=s.voice,
+                accent=s.accent,
+                source=f"{s.source}_hesitation_mid",
+                speed=s.speed,
+            ))
+
+    log.info("Created %d hesitation-pause variants from %d incomplete samples",
+             len(variants), len(incomplete))
+    return variants
+
+
+# ---------------------------------------------------------------------------
+# Short utterance generation (Pipecat v3.2: -40% errors on short responses)
+# ---------------------------------------------------------------------------
+
+# Common short Portuguese responses in meetings
+SHORT_UTTERANCES_COMPLETE = [
+    "Sim.", "Não.", "Ok.", "Tá.", "Pode.", "Beleza.", "Certo.", "Claro.",
+    "Entendi.", "Combinado.", "Perfeito.", "Exato.", "Isso.", "Verdade.",
+    "Com certeza.", "Sem dúvida.", "Tá bom.", "Pode ser.", "Vamos lá.",
+    "Concordo.", "Fechado.", "Ótimo.", "Legal.", "Tranquilo.", "Valeu.",
+    "Obrigado.", "De nada.", "Até logo.", "Tchau.", "Bom dia.", "Boa tarde.",
+    # French speaker variants
+    "Oui... quer dizer, sim.", "Sim, euh, concordo.", "Ok, d'accord.",
+    "Bon, tá bom.", "Voilà, é isso.", "Exactement, exato.",
+]
+
+SHORT_UTTERANCES_INCOMPLETE = [
+    "Sim, mas...", "Não, porque...", "Então...", "Tipo...", "Olha...",
+    "Bom...", "Na verdade...", "Quer dizer...", "Pois é...", "Sabe...",
+    "É que...", "O problema é...", "A questão é...", "Eu acho que...",
+    # French speaker variants (hesitating after short start)
+    "Oui... euh...", "Sim, mas... comment dire...", "Alors...",
+    "Bon, eu acho que... euh...", "C'est... quer dizer...",
+    "Não, enfin... na verdade...", "Sim, mas... como se diz...",
+]
+
+
+def generate_short_utterances(
+    voices: list[str] | None = None,
+    n_per_utterance: int = 3,
+) -> list[AudioSample]:
+    """Generate audio for short utterances using Kokoro TTS.
+
+    Pipecat v3.2 showed that a dedicated short utterance dataset
+    reduced misclassification by 40%. Short responses like "sim", "não"
+    are very common in Portuguese meetings.
+    """
+    try:
+        from kokoro import KPipeline
+    except ImportError:
+        log.warning("kokoro not installed — skipping short utterances")
+        return []
+
+    if voices is None:
+        voices = ["pf_dora", "pm_alex", "pm_santa"]
+
+    pipeline = KPipeline(lang_code="p")
+    samples = []
+    errors = 0
+
+    all_short = (
+        [(text, "complete") for text in SHORT_UTTERANCES_COMPLETE]
+        + [(text, "incomplete") for text in SHORT_UTTERANCES_INCOMPLETE]
+    )
+
+    for text, label in all_short:
+        for _ in range(n_per_utterance):
+            voice = random.choice(voices)
+            speed = random.uniform(0.85, 1.15)
+
+            try:
+                generator = pipeline(text, voice=voice, speed=speed)
+                audio_chunks = []
+                for _, _, chunk in generator:
+                    audio_chunks.append(chunk.numpy() if hasattr(chunk, 'numpy') else np.array(chunk))
+
+                if not audio_chunks:
+                    errors += 1
+                    continue
+
+                audio = np.concatenate(audio_chunks).astype(np.float32)
+
+                # Resample to 16kHz if needed
+                if hasattr(pipeline, 'sample_rate') and pipeline.sample_rate != SAMPLE_RATE:
+                    import torchaudio
+                    import torch
+                    tensor = torch.from_numpy(audio).float().unsqueeze(0)
+                    audio = torchaudio.functional.resample(
+                        tensor, pipeline.sample_rate, SAMPLE_RATE
+                    ).squeeze().numpy()
+
+                # Normalize
+                peak = np.max(np.abs(audio))
+                if peak > 0:
+                    audio = audio / peak * 0.9
+
+                # For incomplete short utterances, add hesitation pause
+                if label == "incomplete" and random.random() < 0.7:
+                    audio = inject_hesitation_pause(audio, pause_duration_range=(1.0, 2.5), position="end")
+
+                samples.append(AudioSample(
+                    audio=audio,
+                    text=text,
+                    label=label,
+                    voice=voice,
+                    accent="native_pt_br",
+                    source="short_utterance",
+                    speed=speed,
+                ))
+
+            except Exception as e:
+                errors += 1
+                if errors <= 5:
+                    log.warning("Short utterance error: %s", e)
+
+    n_c = sum(1 for s in samples if s.label == "complete")
+    n_i = sum(1 for s in samples if s.label == "incomplete")
+    log.info("Short utterances: %d samples (%d complete, %d incomplete, %d errors)",
+             len(samples), n_c, n_i, errors)
+    return samples
+
+
+# ---------------------------------------------------------------------------
+# Audio augmentation
+# ---------------------------------------------------------------------------
+
+def augment_sample(audio: np.ndarray) -> np.ndarray:
+    """Apply augmentation to diversify training data.
+
+    Based on Pipecat/SpeculativeETD augmentation strategies.
+    """
+    aug = audio.copy()
+
+    # Add background noise (simulates meeting room)
+    if random.random() < 0.4:
+        noise_level = random.uniform(0.002, 0.015)
+        aug += np.random.randn(len(aug)).astype(np.float32) * noise_level
+
+    # Volume variation (simulates different mic distances)
+    if random.random() < 0.5:
+        scale = random.uniform(0.6, 1.4)
+        aug *= scale
+
+    # Speed perturbation (simulates speaking rate variation)
+    if random.random() < 0.3:
+        speed = random.uniform(0.92, 1.08)
+        indices = np.arange(0, len(aug), speed).astype(int)
+        indices = indices[indices < len(aug)]
+        aug = aug[indices]
+
+    return np.clip(aug, -1.0, 1.0).astype(np.float32)
+
+
+# ---------------------------------------------------------------------------
+# Save dataset
+# ---------------------------------------------------------------------------
+
+def save_dataset(
+    samples: list[AudioSample],
+    output_dir: Path,
+    augment_copies: int = 2,
+) -> Path:
+    """Save audio samples as WAV files + metadata JSON.
+
+    augment_copies: number of augmented copies per sample (0 = no augmentation)
+    """
+    import soundfile as sf
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    audio_dir = output_dir / "audio"
+    audio_dir.mkdir(exist_ok=True)
+
+    metadata = []
+    total_saved = 0
+
+    for i, sample in enumerate(samples):
+        # Save original
+        fname = f"{i:05d}_{sample.label}_{sample.accent}_{sample.voice}.wav"
+        sf.write(str(audio_dir / fname), sample.audio, SAMPLE_RATE)
+        metadata.append({
+            "file": fname,
+            "text": sample.text,
+            "label": sample.label,
+            "voice": sample.voice,
+            "accent": sample.accent,
+            "source": sample.source,
+            "speed": sample.speed,
+            "augmented": False,
+            "duration_s": round(len(sample.audio) / SAMPLE_RATE, 2),
+        })
+        total_saved += 1
+
+        # Save augmented copies
+        for aug_idx in range(augment_copies):
+            aug_audio = augment_sample(sample.audio)
+            aug_fname = f"{i:05d}_{sample.label}_{sample.accent}_{sample.voice}_aug{aug_idx}.wav"
+            sf.write(str(audio_dir / aug_fname), aug_audio, SAMPLE_RATE)
+            metadata.append({
+                "file": aug_fname,
+                "text": sample.text,
+                "label": sample.label,
+                "voice": sample.voice,
+                "accent": sample.accent,
+                "source": sample.source,
+                "speed": sample.speed,
+                "augmented": True,
+                "duration_s": round(len(aug_audio) / SAMPLE_RATE, 2),
+            })
+            total_saved += 1
+
+    # Save metadata
+    meta_path = output_dir / "metadata.json"
+    with open(meta_path, "w") as f:
+        json.dump(metadata, f, indent=2, ensure_ascii=False)
+
+    # Stats
+    n_complete = sum(1 for m in metadata if m["label"] == "complete")
+    n_incomplete = sum(1 for m in metadata if m["label"] == "incomplete")
+    n_native = sum(1 for m in metadata if m["accent"] == "native_pt_br")
+    n_french = sum(1 for m in metadata if m["accent"] == "french_pt")
+
+    log.info("Dataset saved to %s:", output_dir)
+    log.info("  Total: %d samples (%d original + %d augmented)",
+             total_saved, len(samples), total_saved - len(samples))
+    log.info("  Complete: %d, Incomplete: %d", n_complete, n_incomplete)
+    log.info("  Native PT-BR: %d, French-accented: %d", n_native, n_french)
+
+    return output_dir
+
+
+# ---------------------------------------------------------------------------
+# Main pipeline
+# ---------------------------------------------------------------------------
+
+def run_audio_generation(
+    max_native_sentences: int = 3000,
+    max_french_sentences: int = 1000,
+    augment_copies: int = 2,
+) -> Path:
+    """Run the full audio generation pipeline.
+
+    Loads labeled sentences from 02_generate_labels.py output,
+    generates audio via Kokoro + XTTS, saves dataset.
+    """
+    labeled_dir = DATA_DIR / "claude_labeled"
+
+    # Load labeled sentences
+    all_sentences = []
+
+    # 1. Classified sentences (from CORAA transcripts)
+    classified_path = labeled_dir / "classified_pt.json"
+    if classified_path.exists():
+        with open(classified_path) as f:
+            classified = json.load(f)
+        for c in classified:
+            if c.get("label") in ("completo", "incompleto"):
+                all_sentences.append({
+                    "text": c["text"],
+                    "label": "complete" if c["label"] == "completo" else "incomplete",
+                    "source": "classified",
+                })
+        log.info("Loaded %d classified sentences", len(all_sentences))
+
+    # 2. Filler sentences (all are INCOMPLETE)
+    for filler_file, filler_type in [
+        ("fillers_pt_br.json", "pt_br"),
+        ("fillers_fr_pt.json", "fr_pt"),
+    ]:
+        fpath = labeled_dir / filler_file
+        if fpath.exists():
+            with open(fpath) as f:
+                fillers = json.load(f)
+            for fl in fillers:
+                if fl.get("with_filler"):
+                    all_sentences.append({
+                        "text": fl["with_filler"],
+                        "label": "incomplete",
+                        "source": f"filler_{filler_type}",
+                    })
+            log.info("Loaded %d %s filler sentences", len(fillers), filler_type)
+
+    # 3. French-Portuguese sentences
+    frpt_path = labeled_dir / "french_portuguese.json"
+    if frpt_path.exists():
+        with open(frpt_path) as f:
+            frpt = json.load(f)
+        for s in frpt:
+            label = "complete" if s.get("label", "") == "completo" else "incomplete"
+            all_sentences.append({
+                "text": s["text"],
+                "label": label,
+                "source": "claude_fr_pt",
+            })
+        log.info("Loaded %d French-Portuguese sentences", len(frpt))
+
+    if not all_sentences:
+        log.error("No labeled sentences found in %s — run 02_generate_labels.py first", labeled_dir)
+        return DATA_DIR
+
+    log.info("Total sentences to synthesize: %d", len(all_sentences))
+
+    # Split: native PT-BR vs French-accented
+    native_sentences = [s for s in all_sentences if s["source"] != "claude_fr_pt"]
+    french_sentences = [s for s in all_sentences if s["source"] == "claude_fr_pt"]
+
+    # Also use some filler_fr_pt sentences with XTTS
+    fr_filler = [s for s in all_sentences if s["source"] == "filler_fr_pt"]
+    french_sentences.extend(fr_filler)
+
+    random.shuffle(native_sentences)
+    random.shuffle(french_sentences)
+    native_sentences = native_sentences[:max_native_sentences]
+    french_sentences = french_sentences[:max_french_sentences]
+
+    # Generate audio
+    log.info("\n=== Generating native PT-BR audio (Kokoro) ===")
+    native_samples = generate_kokoro_audio(native_sentences)
+
+    log.info("\n=== Generating French-accented audio (XTTS) ===")
+    ref_dir = DATA_DIR / "french_reference_audio"
+    french_samples = generate_xtts_audio(french_sentences, reference_audio_dir=ref_dir)
+
+    # NEW: Short utterances (Pipecat v3.2: -40% errors)
+    log.info("\n=== Generating short utterance dataset ===")
+    short_samples = generate_short_utterances(n_per_utterance=3)
+
+    # NEW: Hesitation pause variants (SpeculativeETD V3: best data variant)
+    # Critical for French speakers: long pauses mid-sentence are NOT end-of-turn
+    log.info("\n=== Creating hesitation-pause variants ===")
+    all_pre_hesitation = native_samples + french_samples + short_samples
+    hesitation_variants = create_hesitation_variants(all_pre_hesitation, fraction=0.3)
+
+    all_samples = native_samples + french_samples + short_samples + hesitation_variants
+    random.shuffle(all_samples)
+
+    log.info("Total samples before augmentation: %d", len(all_samples))
+    log.info("  Native PT-BR: %d", len(native_samples))
+    log.info("  French-accented: %d", len(french_samples))
+    log.info("  Short utterances: %d", len(short_samples))
+    log.info("  Hesitation variants: %d", len(hesitation_variants))
+
+    # Save
+    log.info("\n=== Saving dataset ===")
+    output_dir = DATA_DIR / "tts_dataset"
+    save_dataset(all_samples, output_dir, augment_copies=augment_copies)
+
+    return output_dir
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s [%(name)s] %(message)s",
+    )
+    random.seed(42)
+    np.random.seed(42)
+
+    run_audio_generation(
+        max_native_sentences=3000,
+        max_french_sentences=1000,
+        augment_copies=2,
+    )

03-finetune-pipecat-pt/04_finetune.py

@@ -0,0 +1,1202 @@
+"""Fine-tune Pipecat Smart Turn v3 for Portuguese + French-accented Portuguese.
+
+Architecture: exact SmartTurnV3Model from Pipecat's train.py
+- WhisperEncoder (openai/whisper-tiny) with max_source_positions=400 (8s)
+- Attention pooling: Linear(384→256) → Tanh → Linear(256→1)
+- Classifier: Linear(384→256) → LayerNorm → GELU → Dropout(0.1)
+                → Linear(256→64) → GELU → Linear(64→1)
+
+Training strategy:
+- Initialize from openai/whisper-tiny (no Pipecat PyTorch weights available)
+- Use Pipecat's Portuguese data as primary training data
+- Add our Claude-labeled + TTS-generated data for PT-BR + French accent
+- Pipecat hyperparams: lr=5e-5, warmup=0.2, cosine schedule, epochs=4
+
+Changes vs initial plan (based on reference analysis — see references/):
+- alpha=0.25 (not 0.6) per Lin et al. 2017 optimal for gamma=2
+- batch_size=128 (not 32) per Pipecat train.py (they use 384)
+- Removed label smoothing (double-regularization with focal loss, per EMNLP 2022)
+- Added BCEWithLogitsLoss option for comparison (Pipecat's original loss)
+- Added real noise augmentation (not just Gaussian) per Pipecat v3.2
+- Added short utterance dataset loading per Pipecat v3.2 findings
+- ONNX opset 18 (not 17) per Pipecat train.py
+- Added INT8 static quantization step per Pipecat's deployment pipeline
+
+Language-learning-specific improvements (March 2026 research):
+- fp_penalty=2.0: asymmetric cost — interrupting a learner costs 2x more than
+  waiting too long (ConversAR 2025, Praktika approach)
+- Speak & Improve L2 corpus: real L2 learner speech with disfluencies (340h)
+- Dual threshold evaluation: eager (speculative) + final (confirmed) per Deepgram Flux
+
+Data sources:
+1. Pipecat v3.2 Portuguese subset (~5K samples from 270K)
+2. Our custom TTS dataset (03_generate_audio.py output)
+3. CORAA real audio (01_download_pipecat.py output)
+4. Speak & Improve L2 corpus (~2K samples, cross-lingual hesitation patterns)
+"""
+
+from __future__ import annotations
+
+import gc
+import json
+import logging
+import random
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset, DataLoader, WeightedRandomSampler
+from transformers import WhisperFeatureExtractor, WhisperPreTrainedModel, WhisperConfig
+
+log = logging.getLogger(__name__)
+
+SAMPLE_RATE = 16000
+WINDOW_SECONDS = 8
+WINDOW_SAMPLES = WINDOW_SECONDS * SAMPLE_RATE
+
+_workspace = Path("/workspace") if Path("/workspace").exists() else Path(".")
+OUTPUT_DIR = _workspace / "results"
+CACHE_DIR = _workspace / "hf_cache"
+DATA_DIR = Path(__file__).parent / "data"
+
+# Label smoothing REMOVED — focal loss already provides implicit calibration
+# via entropy regularization (EMNLP 2022: focal_loss_calibration_emnlp_2022.md)
+# Double-regularization (FL + LS) reduces discriminative power.
+LABEL_SMOOTH = 0.0
+
+
+# ---------------------------------------------------------------------------
+# Model — SmartTurnV3Model (exact Pipecat architecture)
+# ---------------------------------------------------------------------------
+
+class SmartTurnV3Model(nn.Module):
+    """Pipecat Smart Turn v3 model architecture.
+
+    Matches the exact architecture from:
+    https://github.com/pipecat-ai/smart-turn/blob/main/train/train.py
+
+    WhisperEncoder (whisper-tiny, 384-dim) → attention pooling → classifier
+    """
+
+    def __init__(self, whisper_model: str = "openai/whisper-tiny"):
+        super().__init__()
+        from transformers import WhisperModel
+
+        # Load pretrained encoder
+        whisper = WhisperModel.from_pretrained(
+            whisper_model, cache_dir=str(CACHE_DIR)
+        )
+        self.encoder = whisper.encoder
+
+        # Resize position embeddings: 1500 (30s) → 400 (8s)
+        max_pos = 400
+        old_embed = self.encoder.embed_positions.weight.data
+        new_embed = old_embed[:max_pos, :]
+        self.encoder.embed_positions = nn.Embedding(max_pos, old_embed.shape[1])
+        self.encoder.embed_positions.weight.data = new_embed
+        self.encoder.config.max_source_positions = max_pos
+
+        hidden_size = self.encoder.config.d_model  # 384 for whisper-tiny
+
+        # Attention pooling (exact Pipecat architecture)
+        self.attention = nn.Sequential(
+            nn.Linear(hidden_size, 256),
+            nn.Tanh(),
+            nn.Linear(256, 1),
+        )
+
+        # Classifier head (exact Pipecat architecture)
+        self.classifier = nn.Sequential(
+            nn.Linear(hidden_size, 256),
+            nn.LayerNorm(256),
+            nn.GELU(),
+            nn.Dropout(0.1),
+            nn.Linear(256, 64),
+            nn.GELU(),
+            nn.Linear(64, 1),
+        )
+
+    def forward(self, input_features: torch.Tensor) -> torch.Tensor:
+        encoder_output = self.encoder(input_features).last_hidden_state
+        attn_weights = self.attention(encoder_output)
+        attn_weights = torch.softmax(attn_weights, dim=1)
+        pooled = (encoder_output * attn_weights).sum(dim=1)
+        logits = self.classifier(pooled)
+        return logits.squeeze(-1)
+
+
+# ---------------------------------------------------------------------------
+# Focal Loss
+# ---------------------------------------------------------------------------
+
+class FocalLoss(nn.Module):
+    """Focal Loss (Lin et al. 2017) — focuses on hard boundary cases.
+
+    alpha=0.25 is optimal for gamma=2.0 per the original paper (Table 1a).
+    Our initial alpha=0.6 over-weighted positives and hurt calibration.
+
+    fp_penalty: extra multiplier on false-positive loss (model says "complete"
+    when speaker is still talking). For a language-learning avatar, interrupting
+    the learner mid-thought is much worse than waiting too long.
+    - ConversAR (2025) gives learners "infinite thinking period"
+    - Praktika uses extended silence tolerance for L2 speech
+    - Default 2.0 means FP errors cost 2x more than FN errors
+    """
+
+    def __init__(self, gamma: float = 2.0, alpha: float = 0.25, fp_penalty: float = 2.0):
+        super().__init__()
+        self.gamma = gamma
+        self.alpha = alpha
+        self.fp_penalty = fp_penalty
+
+    def forward(self, logits: torch.Tensor, targets: torch.Tensor) -> torch.Tensor:
+        bce = nn.functional.binary_cross_entropy_with_logits(
+            logits, targets, reduction="none",
+        )
+        probs = torch.sigmoid(logits)
+        p_t = probs * targets + (1 - probs) * (1 - targets)
+        focal_weight = (1 - p_t) ** self.gamma
+        alpha_t = self.alpha * targets + (1 - self.alpha) * (1 - targets)
+        loss = alpha_t * focal_weight * bce
+
+        # Asymmetric cost: penalize FP (interrupting learner) more than FN (waiting)
+        # FP = model predicts 1 (complete) when target is 0 (incomplete)
+        if self.fp_penalty != 1.0:
+            is_fp = (probs > 0.5).float() * (1 - targets)  # predicted complete, actually incomplete
+            penalty = 1.0 + (self.fp_penalty - 1.0) * is_fp
+            loss = loss * penalty
+
+        return loss.mean()
+
+
+# ---------------------------------------------------------------------------
+# Data loading
+# ---------------------------------------------------------------------------
+
+@dataclass
+class AudioSample:
+    audio: np.ndarray
+    label: float  # 1.0 = complete, 0.0 = incomplete
+    source: str
+    speaker_id: str = ""
+
+
+def load_pipecat_portuguese(max_samples: int = 5000) -> list[AudioSample]:
+    """Load Pipecat v3.2 Portuguese data (from 01_download_pipecat.py)."""
+    audio_dir = DATA_DIR / "pipecat_pt_audio"
+    if not audio_dir.exists():
+        log.warning("Pipecat PT audio not found at %s — run 01_download_pipecat.py first", audio_dir)
+        return []
+
+    import soundfile as sf
+
+    samples = []
+    for wav_path in sorted(audio_dir.glob("*.wav"))[:max_samples]:
+        try:
+            audio, sr = sf.read(str(wav_path))
+            audio = np.array(audio, dtype=np.float32)
+
+            if sr != SAMPLE_RATE:
+                import torchaudio
+                tensor = torch.from_numpy(audio).float().unsqueeze(0)
+                audio = torchaudio.functional.resample(tensor, sr, SAMPLE_RATE).squeeze().numpy()
+
+            label = 1.0 if "_complete" in wav_path.name else 0.0
+            audio = _extract_window(audio)
+            if audio is not None:
+                samples.append(AudioSample(
+                    audio=audio, label=label,
+                    source="pipecat_v3.2",
+                    speaker_id=f"pipecat_{wav_path.stem.split('_')[0]}",
+                ))
+        except Exception as e:
+            log.warning("Error loading %s: %s", wav_path.name, e)
+
+    n_c = sum(1 for s in samples if s.label == 1.0)
+    n_i = sum(1 for s in samples if s.label == 0.0)
+    log.info("Pipecat PT: %d samples (%d complete, %d incomplete)", len(samples), n_c, n_i)
+    return samples
+
+
+def load_tts_dataset(max_samples: int = 10000) -> list[AudioSample]:
+    """Load TTS-generated audio (from 03_generate_audio.py)."""
+    tts_dir = DATA_DIR / "tts_dataset"
+    meta_path = tts_dir / "metadata.json"
+
+    if not meta_path.exists():
+        log.warning("TTS dataset not found at %s — run 03_generate_audio.py first", tts_dir)
+        return []
+
+    import soundfile as sf
+
+    with open(meta_path) as f:
+        metadata = json.load(f)
+
+    audio_dir = tts_dir / "audio"
+    samples = []
+
+    for meta in metadata[:max_samples]:
+        wav_path = audio_dir / meta["file"]
+        if not wav_path.exists():
+            continue
+
+        try:
+            audio, sr = sf.read(str(wav_path))
+            audio = np.array(audio, dtype=np.float32)
+
+            if sr != SAMPLE_RATE:
+                import torchaudio
+                tensor = torch.from_numpy(audio).float().unsqueeze(0)
+                audio = torchaudio.functional.resample(tensor, sr, SAMPLE_RATE).squeeze().numpy()
+
+            label = 1.0 if meta["label"] == "complete" else 0.0
+            audio = _extract_window(audio)
+            if audio is not None:
+                samples.append(AudioSample(
+                    audio=audio, label=label,
+                    source=f"tts_{meta['accent']}",
+                    speaker_id=meta["voice"],
+                ))
+        except Exception as e:
+            log.warning("Error loading %s: %s", meta["file"], e)
+
+    n_c = sum(1 for s in samples if s.label == 1.0)
+    n_i = sum(1 for s in samples if s.label == 0.0)
+    log.info("TTS dataset: %d samples (%d complete, %d incomplete)", len(samples), n_c, n_i)
+    return samples
+
+
+def load_coraa_real_audio(max_samples: int = 3000) -> list[AudioSample]:
+    """Load REAL conversational audio from CORAA (not TTS).
+
+    SpeculativeETD showed synthetic-only training has a devastating gap:
+    F1 drops from 94.7% → 30.3% on real data. Mixing real audio is critical.
+
+    CORAA MUPE has 365h of real PT-BR interviews with speaker diarization.
+    We use these as COMPLETE samples (full utterances with natural prosody).
+    For INCOMPLETE, we truncate at natural pause points.
+    """
+    from datasets import load_dataset
+    import re
+
+    log.info("Loading CORAA MUPE real audio (streaming)...")
+    try:
+        ds = load_dataset(
+            "nilc-nlp/CORAA-MUPE-ASR",
+            split="train",
+            streaming=True,
+            cache_dir=str(CACHE_DIR),
+        )
+    except Exception as e:
+        log.warning("Failed to load CORAA MUPE: %s", e)
+        return []
+
+    samples = []
+    complete_count = 0
+    incomplete_count = 0
+    target_per_class = max_samples // 2
+
+    for i, row in enumerate(ds):
+        if complete_count >= target_per_class and incomplete_count >= target_per_class:
+            break
+
+        try:
+            audio_data = row.get("audio", {})
+            if not audio_data:
+                continue
+
+            audio = np.array(audio_data["array"], dtype=np.float32)
+            sr = audio_data["sampling_rate"]
+
+            if sr != SAMPLE_RATE:
+                import torchaudio
+                tensor = torch.from_numpy(audio).float().unsqueeze(0)
+                audio = torchaudio.functional.resample(tensor, sr, SAMPLE_RATE).squeeze().numpy()
+
+            duration = len(audio) / SAMPLE_RATE
+            if duration < 1.0:
+                continue
+
+            text = str(row.get("text", ""))
+            speaker_id = f"coraa_real_{i // 50}"
+
+            # COMPLETE: full utterances ending with sentence punctuation
+            if complete_count < target_per_class and re.search(r'[.!?]+\s*$', text):
+                window = _extract_window(audio)
+                if window is not None:
+                    samples.append(AudioSample(
+                        audio=window, label=1.0,
+                        source="coraa_real",
+                        speaker_id=speaker_id,
+                    ))
+                    complete_count += 1
+
+            # INCOMPLETE: truncate real audio at 40-70% (natural mid-utterance)
+            if incomplete_count < target_per_class and duration >= 2.0:
+                cut_frac = random.uniform(0.4, 0.7)
+                truncated = audio[:int(len(audio) * cut_frac)]
+                window = _extract_window(truncated)
+                if window is not None:
+                    samples.append(AudioSample(
+                        audio=window, label=0.0,
+                        source="coraa_real",
+                        speaker_id=speaker_id,
+                    ))
+                    incomplete_count += 1
+
+        except Exception:
+            continue
+
+        if i % 5000 == 0 and i > 0:
+            log.info("  CORAA real: scanned %d, %d complete, %d incomplete", i, complete_count, incomplete_count)
+
+    log.info("CORAA real audio: %d complete + %d incomplete = %d samples",
+             complete_count, incomplete_count, len(samples))
+    return samples
+
+
+def load_pipecat_test_data() -> list[AudioSample]:
+    """Load Pipecat v3.2 Portuguese test data for evaluation."""
+    test_dir = DATA_DIR / "pipecat_pt_test"
+    if not test_dir.exists():
+        log.warning("Pipecat PT test data not found — run 01_download_pipecat.py first")
+        return []
+
+    import soundfile as sf
+
+    samples = []
+    for wav_path in sorted(test_dir.glob("*.wav")):
+        try:
+            audio, sr = sf.read(str(wav_path))
+            audio = np.array(audio, dtype=np.float32)
+
+            if sr != SAMPLE_RATE:
+                import torchaudio
+                tensor = torch.from_numpy(audio).float().unsqueeze(0)
+                audio = torchaudio.functional.resample(tensor, sr, SAMPLE_RATE).squeeze().numpy()
+
+            label = 1.0 if "_complete" in wav_path.name else 0.0
+            audio = _extract_window(audio)
+            if audio is not None:
+                samples.append(AudioSample(
+                    audio=audio, label=label,
+                    source="pipecat_test",
+                    speaker_id=f"pipecat_test_{wav_path.stem.split('_')[0]}",
+                ))
+        except Exception as e:
+            log.warning("Error loading test %s: %s", wav_path.name, e)
+
+    log.info("Pipecat test: %d samples", len(samples))
+    return samples
+
+
+def _extract_window(audio: np.ndarray) -> np.ndarray | None:
+    """Extract 8-second window from end of audio, pad if needed."""
+    if len(audio) < SAMPLE_RATE:  # minimum 1s
+        return None
+
+    peak = np.max(np.abs(audio))
+    if peak > 0:
+        audio = audio / peak * 0.9
+
+    if len(audio) > WINDOW_SAMPLES:
+        audio = audio[-WINDOW_SAMPLES:]
+    elif len(audio) < WINDOW_SAMPLES:
+        padding = WINDOW_SAMPLES - len(audio)
+        audio = np.pad(audio, (padding, 0), mode="constant")
+
+    # ~200ms silence at end (VAD behavior)
+    silence_samples = int(0.2 * SAMPLE_RATE)
+    audio[-silence_samples:] = 0.0
+
+    return audio.astype(np.float32)
+
+
+# ---------------------------------------------------------------------------
+# Audio augmentation
+# ---------------------------------------------------------------------------
+
+# Cache for background noise samples (loaded once)
+_noise_cache: list[np.ndarray] = []
+
+
+def _load_noise_samples() -> list[np.ndarray]:
+    """Load real background noise samples for augmentation.
+
+    Pipecat v3.2 used CC-0 Freesound.org cafe/office noise and saw
+    40% fewer short-utterance misclassifications (pipecat_smart_turn_v3_2.md).
+    """
+    global _noise_cache
+    if _noise_cache:
+        return _noise_cache
+
+    noise_dir = DATA_DIR / "noise_samples"
+    if not noise_dir.exists():
+        return []
+
+    import soundfile as sf
+    for wav_path in noise_dir.glob("*.wav"):
+        try:
+            audio, sr = sf.read(str(wav_path))
+            audio = np.array(audio, dtype=np.float32)
+            if sr != SAMPLE_RATE:
+                import torchaudio
+                tensor = torch.from_numpy(audio).float().unsqueeze(0)
+                audio = torchaudio.functional.resample(tensor, sr, SAMPLE_RATE).squeeze().numpy()
+            _noise_cache.append(audio)
+        except Exception:
+            pass
+
+    if _noise_cache:
+        log.info("Loaded %d background noise samples for augmentation", len(_noise_cache))
+    return _noise_cache
+
+
+def augment_audio(audio: np.ndarray) -> np.ndarray:
+    """Data augmentation for training.
+
+    Per reference analysis:
+    - Real background noise (Pipecat v3.2: 40% fewer errors with cafe/office noise)
+    - Speed perturbation (standard in speech ML)
+    - Volume variation (simulates mic distance)
+    - Time shift
+    """
+    aug = audio.copy()
+
+    # Speed perturbation
+    if random.random() < 0.5:
+        speed = random.uniform(0.9, 1.1)
+        indices = np.arange(0, len(aug), speed).astype(int)
+        indices = indices[indices < len(aug)]
+        aug = aug[indices]
+        if len(aug) > WINDOW_SAMPLES:
+            aug = aug[:WINDOW_SAMPLES]
+        elif len(aug) < WINDOW_SAMPLES:
+            aug = np.pad(aug, (WINDOW_SAMPLES - len(aug), 0), mode="constant")
+
+    # Volume variation
+    if random.random() < 0.5:
+        aug *= random.uniform(0.6, 1.4)
+
+    # Real background noise (preferred) or Gaussian fallback
+    noise_samples = _load_noise_samples()
+    if noise_samples and random.random() < 0.5:
+        noise = random.choice(noise_samples)
+        # Loop or trim noise to match audio length
+        if len(noise) < len(aug):
+            repeats = len(aug) // len(noise) + 1
+            noise = np.tile(noise, repeats)[:len(aug)]
+        else:
+            start = random.randint(0, len(noise) - len(aug))
+            noise = noise[start:start + len(aug)]
+        snr_db = random.uniform(10, 25)  # 10-25 dB SNR
+        noise_scale = np.sqrt(np.mean(aug ** 2)) / (np.sqrt(np.mean(noise ** 2)) * 10 ** (snr_db / 20) + 1e-8)
+        aug += noise * noise_scale
+    elif random.random() < 0.4:
+        # Gaussian noise fallback
+        noise_level = random.uniform(0.002, 0.02)
+        aug += np.random.randn(len(aug)).astype(np.float32) * noise_level
+
+    # Time shift
+    if random.random() < 0.3:
+        shift = random.randint(-int(0.3 * SAMPLE_RATE), int(0.3 * SAMPLE_RATE))
+        aug = np.roll(aug, shift)
+        if shift > 0:
+            aug[:shift] = 0.0
+        elif shift < 0:
+            aug[shift:] = 0.0
+
+    return np.clip(aug, -1.0, 1.0).astype(np.float32)
+
+
+# ---------------------------------------------------------------------------
+# PyTorch Dataset
+# ---------------------------------------------------------------------------
+
+class SmartTurnDataset(Dataset):
+    def __init__(
+        self,
+        samples: list[AudioSample],
+        feature_extractor: WhisperFeatureExtractor,
+        augment: bool = False,
+    ):
+        self.samples = samples
+        self.feature_extractor = feature_extractor
+        self.augment = augment
+
+    def __len__(self) -> int:
+        return len(self.samples)
+
+    def __getitem__(self, idx: int) -> dict:
+        sample = self.samples[idx]
+        audio = sample.audio
+
+        if self.augment:
+            audio = augment_audio(audio)
+
+        inputs = self.feature_extractor(
+            audio,
+            sampling_rate=SAMPLE_RATE,
+            return_tensors="np",
+            padding="max_length",
+            max_length=WINDOW_SAMPLES,
+            truncation=True,
+            do_normalize=True,
+        )
+
+        features = inputs.input_features.squeeze(0).astype(np.float32)
+
+        return {
+            "input_features": torch.from_numpy(features),
+            "labels": torch.tensor(sample.label, dtype=torch.float32),
+        }
+
+
+# ---------------------------------------------------------------------------
+# Train/val split
+# ---------------------------------------------------------------------------
+
+def split_by_speaker(
+    samples: list[AudioSample],
+    val_frac: float = 0.1,
+    test_frac: float = 0.1,
+) -> tuple[list[AudioSample], list[AudioSample], list[AudioSample]]:
+    """Split by speaker ID to prevent data leakage."""
+    speaker_map: dict[str, list[AudioSample]] = {}
+    for s in samples:
+        speaker_map.setdefault(s.speaker_id or "unknown", []).append(s)
+
+    speakers = list(speaker_map.keys())
+    random.shuffle(speakers)
+
+    n_val = max(1, int(len(speakers) * val_frac))
+    n_test = max(1, int(len(speakers) * test_frac))
+
+    test_spk = set(speakers[:n_test])
+    val_spk = set(speakers[n_test:n_test + n_val])
+    train_spk = set(speakers[n_test + n_val:])
+
+    train = [s for sp in train_spk for s in speaker_map[sp]]
+    val = [s for sp in val_spk for s in speaker_map[sp]]
+    test = [s for sp in test_spk for s in speaker_map[sp]]
+
+    random.shuffle(train)
+    random.shuffle(val)
+    random.shuffle(test)
+
+    for name, split in [("Train", train), ("Val", val), ("Test", test)]:
+        n_c = sum(1 for s in split if s.label == 1.0)
+        n_i = sum(1 for s in split if s.label == 0.0)
+        log.info("  %s: %d samples (%d complete, %d incomplete, %d speakers)",
+                 name, len(split), n_c, n_i, len(set(s.speaker_id for s in split)))
+
+    return train, val, test
+
+
+# ---------------------------------------------------------------------------
+# Training
+# ---------------------------------------------------------------------------
+
+def load_speak_improve_l2(max_samples: int = 2000) -> list[AudioSample]:
+    """Load L2 learner speech from Speak & Improve Corpus 2025.
+
+    340 hours of L2 English learner speech with disfluency annotations and
+    CEFR proficiency scores (A2-C1, majority B1-B2).
+
+    Even though this is English L2 (not Portuguese), L2 hesitation patterns
+    transfer across languages (Cenoz 2000). The pauses, false starts, and
+    disfluencies of L2 speakers share universal characteristics regardless
+    of target language.
+
+    Knill et al. (2025). Speak & Improve Corpus 2025: an L2 English Speech
+    Corpus for Language Assessment and Feedback. arXiv:2412.11986
+    """
+    from datasets import load_dataset
+
+    log.info("Loading Speak & Improve L2 corpus (streaming)...")
+    try:
+        ds = load_dataset(
+            "CambridgeEnglish/SpeakAndImprove2025",
+            split="train",
+            streaming=True,
+            cache_dir=str(CACHE_DIR),
+            trust_remote_code=True,
+        )
+    except Exception as e:
+        log.warning("Failed to load Speak & Improve corpus: %s — trying fallback name", e)
+        try:
+            ds = load_dataset(
+                "cambridgeenglishtests/speak-and-improve-2025",
+                split="train",
+                streaming=True,
+                cache_dir=str(CACHE_DIR),
+                trust_remote_code=True,
+            )
+        except Exception as e2:
+            log.warning("Speak & Improve corpus not available: %s — skipping", e2)
+            return []
+
+    samples = []
+    complete_count = 0
+    incomplete_count = 0
+    target_per_class = max_samples // 2
+
+    for i, row in enumerate(ds):
+        if complete_count >= target_per_class and incomplete_count >= target_per_class:
+            break
+
+        try:
+            audio_data = row.get("audio", {})
+            if not audio_data:
+                continue
+
+            audio = np.array(audio_data["array"], dtype=np.float32)
+            sr = audio_data["sampling_rate"]
+
+            if sr != SAMPLE_RATE:
+                import torchaudio
+                tensor = torch.from_numpy(audio).float().unsqueeze(0)
+                audio = torchaudio.functional.resample(tensor, sr, SAMPLE_RATE).squeeze().numpy()
+
+            duration = len(audio) / SAMPLE_RATE
+            if duration < 1.0:
+                continue
+
+            text = str(row.get("text", row.get("transcript", "")))
+            speaker_id = f"si_l2_{i // 20}"
+
+            # COMPLETE: full utterances (use entire audio)
+            if complete_count < target_per_class and duration >= 2.0:
+                window = _extract_window(audio)
+                if window is not None:
+                    samples.append(AudioSample(
+                        audio=window, label=1.0,
+                        source="speak_improve_l2",
+                        speaker_id=speaker_id,
+                    ))
+                    complete_count += 1
+
+            # INCOMPLETE: truncate at random point (simulates mid-utterance)
+            if incomplete_count < target_per_class and duration >= 3.0:
+                cut_frac = random.uniform(0.3, 0.6)
+                truncated = audio[:int(len(audio) * cut_frac)]
+                # Add hesitation-like pause at end (L2 speakers pause 524ms+ per Kosmala 2022)
+                pause_ms = random.uniform(500, 2000)
+                pause_samples = int(pause_ms / 1000 * SAMPLE_RATE)
+                pause = np.random.randn(pause_samples).astype(np.float32) * 0.001
+                truncated = np.concatenate([truncated, pause])
+                window = _extract_window(truncated)
+                if window is not None:
+                    samples.append(AudioSample(
+                        audio=window, label=0.0,
+                        source="speak_improve_l2",
+                        speaker_id=speaker_id,
+                    ))
+                    incomplete_count += 1
+
+        except Exception:
+            continue
+
+        if i % 2000 == 0 and i > 0:
+            log.info("  S&I L2: scanned %d, %d complete, %d incomplete", i, complete_count, incomplete_count)
+
+    log.info("Speak & Improve L2: %d complete + %d incomplete = %d samples",
+             complete_count, incomplete_count, len(samples))
+    return samples
+
+
+def train(
+    epochs: int = 6,
+    batch_size: int = 128,
+    lr: float = 5e-5,
+    warmup_ratio: float = 0.2,
+    max_pipecat_samples: int = 5000,
+    max_tts_samples: int = 10000,
+    max_l2_samples: int = 2000,
+    whisper_model: str = "openai/whisper-tiny",
+    loss_fn: str = "focal",  # "focal" or "bce" (Pipecat's original)
+    fp_penalty: float = 2.0,  # asymmetric cost: FP costs 2x more than FN
+) -> Path:
+    """Fine-tune SmartTurnV3Model on Portuguese data.
+
+    Default hyperparams from Pipecat's train.py:
+    - lr=5e-5, warmup_ratio=0.2, cosine schedule
+    - Pipecat uses epochs=4 on 270K samples; we use 6 on 5-15K (more passes needed)
+    - Pipecat uses batch_size=384; we use 128 (A10G has 24GB, enough for 128)
+    - loss_fn="focal" (our improvement) or "bce" (Pipecat's original) for comparison
+
+    Language-learning-specific improvements (March 2026 research):
+    - fp_penalty=2.0: asymmetric cost — interrupting a learner mid-thought is
+      much worse than waiting too long (ConversAR 2025, Praktika approach)
+    - Speak & Improve L2 corpus: real L2 learner speech with hesitations/disfluencies
+    - Dual threshold in evaluation: eager (0.3-0.5) for speculative prep,
+      final (0.7+) for actual turn transition (inspired by Deepgram Flux)
+    """
+
+    device = "cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")
+    log.info("Training on device: %s", device)
+    if device == "cuda":
+        log.info("GPU: %s (%d MB)", torch.cuda.get_device_name(),
+                 torch.cuda.get_device_properties(0).total_memory // 1024 // 1024)
+
+    # ----- Load all data sources -----
+    t0 = time.time()
+    all_samples: list[AudioSample] = []
+
+    # 1. Pipecat Portuguese (primary — these have LLM-curated labels)
+    pipecat = load_pipecat_portuguese(max_samples=max_pipecat_samples)
+    all_samples.extend(pipecat)
+    del pipecat
+    gc.collect()
+
+    # 2. Our TTS-generated data (native PT-BR + French accent + short utterances)
+    tts = load_tts_dataset(max_samples=max_tts_samples)
+    all_samples.extend(tts)
+    del tts
+    gc.collect()
+
+    # 3. CORAA real audio (critical: SpeculativeETD showed synthetic→real gap of 94.7%→30.3%)
+    coraa = load_coraa_real_audio(max_samples=3000)
+    all_samples.extend(coraa)
+    del coraa
+    gc.collect()
+
+    # 4. Speak & Improve L2 corpus — real L2 learner speech with disfluencies
+    # L2 hesitation patterns transfer across languages (Cenoz 2000)
+    # 340h of L2 English learners, CEFR A2-C1 (arXiv:2412.11986)
+    si_l2 = load_speak_improve_l2(max_samples=max_l2_samples)
+    all_samples.extend(si_l2)
+    del si_l2
+    gc.collect()
+
+    if not all_samples:
+        raise RuntimeError("No training samples loaded! Run 01/03 scripts first.")
+
+    load_time = time.time() - t0
+    n_c = sum(1 for s in all_samples if s.label == 1.0)
+    n_i = sum(1 for s in all_samples if s.label == 0.0)
+    sources = {}
+    for s in all_samples:
+        sources[s.source] = sources.get(s.source, 0) + 1
+
+    log.info("Total: %d samples (%d complete, %d incomplete) in %.0fs", len(all_samples), n_c, n_i, load_time)
+    for src, cnt in sorted(sources.items()):
+        log.info("  %s: %d", src, cnt)
+
+    # ----- Split -----
+    log.info("=== Splitting by speaker ===")
+    train_samples, val_samples, internal_test = split_by_speaker(all_samples)
+
+    # Also load Pipecat test data (held-out, never seen during training)
+    pipecat_test = load_pipecat_test_data()
+
+    # ----- Datasets -----
+    feature_extractor = WhisperFeatureExtractor(chunk_length=8)
+    train_ds = SmartTurnDataset(train_samples, feature_extractor, augment=True)
+    val_ds = SmartTurnDataset(val_samples, feature_extractor, augment=False)
+
+    # Balanced sampling
+    train_labels = [s.label for s in train_samples]
+    n_pos = sum(1 for l in train_labels if l == 1.0)
+    n_neg = len(train_labels) - n_pos
+    weights = [1.0 / n_neg if l == 0.0 else 1.0 / n_pos for l in train_labels]
+    sampler = WeightedRandomSampler(weights, len(weights))
+
+    use_pin = device == "cuda"
+    n_workers = 4 if device == "cuda" else 0
+    train_loader = DataLoader(train_ds, batch_size=batch_size, sampler=sampler,
+                              num_workers=n_workers, pin_memory=use_pin)
+    val_loader = DataLoader(val_ds, batch_size=batch_size, shuffle=False,
+                            num_workers=0, pin_memory=use_pin)
+
+    # ----- Model -----
+    model = SmartTurnV3Model(whisper_model=whisper_model).to(device)
+    total_params = sum(p.numel() for p in model.parameters())
+    log.info("Model: %d params (%.1f MB)", total_params, total_params * 4 / 1024 / 1024)
+
+    # ----- Loss -----
+    # Pipecat uses BCEWithLogitsLoss with dynamic pos_weight (clamped 0.1-10.0)
+    # We default to Focal Loss (alpha=0.25, gamma=2.0 per Lin et al. 2017)
+    # but support BCE for comparison
+    pos_weight_val = min(max(n_neg / max(n_pos, 1), 0.1), 10.0)
+    pos_weight = torch.tensor([pos_weight_val], device=device)
+
+    if loss_fn == "bce":
+        criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight)
+        log.info("Loss: BCEWithLogitsLoss (Pipecat original), pos_weight=%.2f", pos_weight_val)
+    else:
+        criterion = FocalLoss(gamma=2.0, alpha=0.25, fp_penalty=fp_penalty)
+        log.info("Loss: FocalLoss (gamma=2.0, alpha=0.25, fp_penalty=%.1f)", fp_penalty)
+        log.info("  Asymmetric cost: FP (interrupting learner) penalized %.1fx more than FN", fp_penalty)
+
+    # ----- Optimizer: uniform LR (Pipecat uses single lr=5e-5 for all params) -----
+    # Pipecat's train.py does NOT use differential LR — all params at same rate.
+    # Since we initialize from whisper-tiny (same as Pipecat), uniform LR is correct.
+    optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=0.01)
+
+    # Cosine schedule with warmup (Pipecat: warmup_ratio=0.2)
+    total_steps = epochs * len(train_loader)
+    warmup_steps = int(total_steps * warmup_ratio)
+
+    def lr_lambda(step):
+        if step < warmup_steps:
+            return step / max(warmup_steps, 1)
+        progress = (step - warmup_steps) / max(total_steps - warmup_steps, 1)
+        return 0.5 * (1 + np.cos(np.pi * progress))
+
+    scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)
+
+    # ----- Training loop -----
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+    best_f1 = 0.0
+    best_path = OUTPUT_DIR / "best_model.pt"
+    resume_path = OUTPUT_DIR / "resume_checkpoint.pt"
+    patience = 5
+    patience_counter = 0
+    history = []
+    start_epoch = 0
+
+    # Resume support
+    if resume_path.exists():
+        log.info("=== Resuming from checkpoint ===")
+        ckpt = torch.load(resume_path, map_location=device, weights_only=False)
+        model.load_state_dict(ckpt["model_state_dict"])
+        optimizer.load_state_dict(ckpt["optimizer_state_dict"])
+        scheduler.load_state_dict(ckpt["scheduler_state_dict"])
+        start_epoch = ckpt["epoch"]
+        best_f1 = ckpt.get("best_f1", 0.0)
+        patience_counter = ckpt.get("patience_counter", 0)
+        history = ckpt.get("history", [])
+        log.info("  Resumed at epoch %d, best_f1=%.4f", start_epoch, best_f1)
+
+    log.info("=== Training: %d epochs, batch=%d, lr=%.1e, warmup_ratio=%.1f ===",
+             epochs, batch_size, lr, warmup_ratio)
+
+    for epoch in range(start_epoch, epochs):
+        model.train()
+        train_loss = 0.0
+        train_correct = 0
+        train_total = 0
+        t_epoch = time.time()
+
+        for batch_idx, batch in enumerate(train_loader):
+            features = batch["input_features"].to(device)
+            labels = batch["labels"].to(device)
+
+            logits = model(features)
+            loss = criterion(logits, labels)
+
+            optimizer.zero_grad()
+            loss.backward()
+            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+            optimizer.step()
+            scheduler.step()
+
+            train_loss += loss.item() * len(labels)
+            preds = (torch.sigmoid(logits) > 0.5).float()
+            hard_labels = (labels > 0.5).float()
+            train_correct += (preds == hard_labels).sum().item()
+            train_total += len(labels)
+
+            if batch_idx % 50 == 0 and batch_idx > 0:
+                log.info("  batch %d/%d loss=%.4f", batch_idx, len(train_loader), loss.item())
+
+        # Validate
+        model.eval()
+        val_metrics = _evaluate(model, val_loader, device, criterion)
+        train_acc = train_correct / max(train_total, 1)
+        epoch_time = time.time() - t_epoch
+
+        log.info(
+            "Epoch %d/%d (%.0fs): train_loss=%.4f train_acc=%.3f | "
+            "val_acc=%.3f val_f1=%.3f prec=%.3f rec=%.3f",
+            epoch + 1, epochs, epoch_time,
+            train_loss / max(train_total, 1), train_acc,
+            val_metrics["accuracy"], val_metrics["f1"],
+            val_metrics["precision"], val_metrics["recall"],
+        )
+
+        history.append({
+            "epoch": epoch + 1,
+            "train_loss": train_loss / max(train_total, 1),
+            "train_acc": train_acc,
+            **{f"val_{k}": v for k, v in val_metrics.items()},
+        })
+
+        # Save best
+        if val_metrics["f1"] > best_f1:
+            best_f1 = val_metrics["f1"]
+            torch.save({
+                "model_state_dict": model.state_dict(),
+                "epoch": epoch + 1,
+                "val_f1": best_f1,
+                "val_metrics": val_metrics,
+                "whisper_model": whisper_model,
+            }, best_path)
+            log.info("  -> New best model (val_f1=%.4f)", best_f1)
+            patience_counter = 0
+        else:
+            patience_counter += 1
+            if patience_counter >= patience:
+                log.info("Early stopping at epoch %d", epoch + 1)
+                break
+
+        # Resume checkpoint
+        torch.save({
+            "model_state_dict": model.state_dict(),
+            "optimizer_state_dict": optimizer.state_dict(),
+            "scheduler_state_dict": scheduler.state_dict(),
+            "epoch": epoch + 1,
+            "best_f1": best_f1,
+            "patience_counter": patience_counter,
+            "history": history,
+        }, resume_path)
+
+    if resume_path.exists():
+        resume_path.unlink()
+
+    # ----- Final evaluation on internal test split -----
+    log.info("\n=== Internal Test Evaluation ===")
+    checkpoint = torch.load(best_path, map_location=device, weights_only=True)
+    model.load_state_dict(checkpoint["model_state_dict"])
+    model.eval()
+
+    if internal_test:
+        test_ds = SmartTurnDataset(internal_test, feature_extractor, augment=False)
+        test_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False)
+        test_metrics = _evaluate(model, test_loader, device, criterion)
+        _log_metrics("Internal test", test_metrics)
+    else:
+        test_metrics = {}
+
+    # ----- Pipecat test set (baseline comparison) -----
+    pipecat_test_metrics = {}
+    if pipecat_test:
+        log.info("\n=== Pipecat PT Test Evaluation (baseline comparison) ===")
+        pipecat_ds = SmartTurnDataset(pipecat_test, feature_extractor, augment=False)
+        pipecat_loader = DataLoader(pipecat_ds, batch_size=batch_size, shuffle=False)
+        pipecat_test_metrics = _evaluate(model, pipecat_loader, device, criterion)
+        _log_metrics("Pipecat PT test", pipecat_test_metrics)
+        log.info("  Pipecat baseline: 95.42%% accuracy, 2.79%% FP, 1.79%% FN")
+
+    # ----- Threshold sweep -----
+    log.info("\n=== Threshold Sweep ===")
+    threshold_results = {}
+    eval_loader = test_loader if internal_test else val_loader
+    for thresh in [0.3, 0.4, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85]:
+        t_metrics = _evaluate(model, eval_loader, device, criterion, threshold=thresh)
+        threshold_results[str(thresh)] = t_metrics
+        log.info("  threshold=%.2f: prec=%.3f rec=%.3f f1=%.3f acc=%.3f FP=%.3f",
+                 thresh, t_metrics["precision"], t_metrics["recall"],
+                 t_metrics["f1"], t_metrics["accuracy"], t_metrics["fp_rate"])
+
+    # ----- Dual threshold recommendation (inspired by Deepgram Flux) -----
+    # For a language-learning avatar:
+    # - eager_threshold (0.3-0.5): start preparing response speculatively
+    #   (e.g., begin LLM generation) — reduces perceived latency
+    # - final_threshold (0.7+): actually take the turn and speak
+    #   Higher final threshold = fewer interruptions (critical for L2 learners)
+    log.info("\n=== Dual Threshold Recommendation (Deepgram Flux-inspired) ===")
+    # Find final threshold: maximize precision with recall >= 85%
+    # (we want very few interruptions, even at the cost of some missed turns)
+    final_candidates = [(k, v) for k, v in threshold_results.items()
+                        if v["recall"] >= 0.85 and float(k) >= 0.6]
+    if final_candidates:
+        best_final = min(final_candidates, key=lambda x: x[1]["fp_rate"])
+        log.info("  Recommended final_threshold: %.2f (FP=%.1f%%, prec=%.3f, rec=%.3f)",
+                 float(best_final[0]), best_final[1]["fp_rate"] * 100,
+                 best_final[1]["precision"], best_final[1]["recall"])
+    else:
+        best_final = ("0.7", threshold_results.get("0.7", {}))
+        log.info("  Default final_threshold: 0.70")
+
+    # Eager threshold: lower confidence where we start speculative processing
+    eager_thresh = max(0.3, float(best_final[0]) - 0.3)
+    log.info("  Recommended eager_threshold: %.2f (start LLM generation speculatively)",
+             eager_thresh)
+    log.info("  Latency savings: ~150-250ms earlier response start (Deepgram Flux benchmark)")
+
+    # ----- ONNX export (FP32 → INT8, per Pipecat's deploy pipeline) -----
+    model = model.to("cpu")
+    onnx_fp32_path = OUTPUT_DIR / "smart_turn_pt_v3_fp32.onnx"
+    onnx_int8_path = OUTPUT_DIR / "smart_turn_pt_v3.onnx"
+    dummy = torch.randn(1, 80, 800)
+    try:
+        # Step 1: Export FP32 ONNX (opset 18, per Pipecat train.py)
+        torch.onnx.export(
+            model, dummy, str(onnx_fp32_path),
+            input_names=["input_features"],
+            output_names=["logits"],
+            dynamic_axes={"input_features": {0: "batch"}, "logits": {0: "batch"}},
+            opset_version=18,
+        )
+        log.info("ONNX FP32 exported: %s (%.1f MB)", onnx_fp32_path,
+                 onnx_fp32_path.stat().st_size / 1024 / 1024)
+
+        # Step 2: INT8 static quantization (entropy calibration, per Pipecat)
+        # Pipecat uses: 1024 calibration samples, QDQ format, Entropy method
+        try:
+            from onnxruntime.quantization import quantize_static, CalibrationDataReader, QuantFormat
+
+            class SmartTurnCalibrationReader(CalibrationDataReader):
+                def __init__(self, dataset, n_samples=1024):
+                    self.data = []
+                    for i, sample in enumerate(dataset):
+                        if i >= n_samples:
+                            break
+                        self.data.append({"input_features": sample["input_features"].unsqueeze(0).numpy()})
+                    self.idx = 0
+
+                def get_next(self):
+                    if self.idx >= len(self.data):
+                        return None
+                    result = self.data[self.idx]
+                    self.idx += 1
+                    return result
+
+            # Use validation data for calibration
+            calib_reader = SmartTurnCalibrationReader(val_ds, n_samples=1024)
+            quantize_static(
+                str(onnx_fp32_path),
+                str(onnx_int8_path),
+                calib_reader,
+                quant_format=QuantFormat.QDQ,
+            )
+            log.info("ONNX INT8 exported: %s (%.1f MB)", onnx_int8_path,
+                     onnx_int8_path.stat().st_size / 1024 / 1024)
+        except ImportError:
+            log.warning("onnxruntime.quantization not available — saving FP32 only")
+            import shutil
+            shutil.copy2(onnx_fp32_path, onnx_int8_path)
+        except Exception as e:
+            log.warning("INT8 quantization failed: %s — saving FP32 only", e)
+            import shutil
+            shutil.copy2(onnx_fp32_path, onnx_int8_path)
+
+    except Exception as e:
+        log.warning("ONNX export failed: %s", e)
+
+    # ----- Save results -----
+    results = {
+        "model": "smart_turn_pt_v3_finetuned",
+        "whisper_model": whisper_model,
+        "architecture": "SmartTurnV3Model (exact Pipecat)",
+        "target_domain": "language_learning_avatar",
+        "learner_profile": "francophone_learning_portuguese",
+        "total_samples": len(all_samples),
+        "sources": sources,
+        "best_epoch": checkpoint["epoch"],
+        "best_val_f1": best_f1,
+        "test_metrics": test_metrics,
+        "pipecat_test_metrics": pipecat_test_metrics,
+        "pipecat_baseline": {"accuracy": 0.9542, "fp_rate": 0.0279, "fn_rate": 0.0179},
+        "threshold_sweep": threshold_results,
+        "dual_threshold": {
+            "eager_threshold": eager_thresh,
+            "final_threshold": float(best_final[0]),
+            "rationale": "For L2 language learning: higher final threshold reduces "
+                        "interruptions. Eager threshold enables speculative LLM "
+                        "generation 150-250ms earlier (Deepgram Flux approach).",
+        },
+        "history": history,
+        "config": {
+            "epochs": epochs,
+            "batch_size": batch_size,
+            "lr": lr,
+            "warmup_ratio": warmup_ratio,
+            "label_smoothing": LABEL_SMOOTH,
+            "focal_loss_gamma": 2.0,
+            "focal_loss_alpha": 0.25,
+            "fp_penalty": fp_penalty,
+            "loss_fn": loss_fn,
+        },
+    }
+
+    results_path = OUTPUT_DIR / "training_results.json"
+    with open(results_path, "w") as f:
+        json.dump(results, f, indent=2)
+    log.info("Results saved to %s", results_path)
+
+    return onnx_int8_path
+
+
+def _evaluate(
+    model: nn.Module,
+    loader: DataLoader,
+    device: str,
+    criterion: nn.Module,
+    threshold: float = 0.5,
+) -> dict:
+    """Evaluate model at a given threshold."""
+    correct = total = tp = fp = fn = tn = 0
+    total_loss = 0.0
+
+    with torch.no_grad():
+        for batch in loader:
+            features = batch["input_features"].to(device)
+            labels = batch["labels"].to(device)
+            logits = model(features)
+            loss = criterion(logits, labels)
+            total_loss += loss.item() * len(labels)
+
+            preds = (torch.sigmoid(logits) > threshold).float()
+            hard_labels = (labels > 0.5).float()
+            correct += (preds == hard_labels).sum().item()
+            total += len(labels)
+            tp += ((preds == 1) & (hard_labels == 1)).sum().item()
+            fp += ((preds == 1) & (hard_labels == 0)).sum().item()
+            fn += ((preds == 0) & (hard_labels == 1)).sum().item()
+            tn += ((preds == 0) & (hard_labels == 0)).sum().item()
+
+    accuracy = correct / max(total, 1)
+    precision = tp / max(tp + fp, 1)
+    recall = tp / max(tp + fn, 1)
+    f1 = 2 * precision * recall / max(precision + recall, 1e-8)
+
+    return {
+        "accuracy": round(accuracy, 4),
+        "precision": round(precision, 4),
+        "recall": round(recall, 4),
+        "f1": round(f1, 4),
+        "loss": round(total_loss / max(total, 1), 4),
+        "tp": tp, "fp": fp, "fn": fn, "tn": tn,
+        "fp_rate": round(fp / max(fp + tn, 1), 4),
+        "fn_rate": round(fn / max(fn + tp, 1), 4),
+    }
+
+
+def _log_metrics(name: str, metrics: dict) -> None:
+    log.info("%s results:", name)
+    log.info("  Accuracy:  %.3f", metrics["accuracy"])
+    log.info("  Precision: %.3f", metrics["precision"])
+    log.info("  Recall:    %.3f", metrics["recall"])
+    log.info("  F1:        %.3f", metrics["f1"])
+    log.info("  FP rate:   %.3f", metrics["fp_rate"])
+    log.info("  FN rate:   %.3f", metrics["fn_rate"])
+    log.info("  TP=%d FP=%d FN=%d TN=%d", metrics["tp"], metrics["fp"], metrics["fn"], metrics["tn"])
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s [%(name)s] %(message)s",
+    )
+    random.seed(42)
+    np.random.seed(42)
+    torch.manual_seed(42)
+
+    train(
+        epochs=6,
+        batch_size=128,
+        lr=5e-5,
+        warmup_ratio=0.2,
+        max_pipecat_samples=5000,
+        max_tts_samples=10000,
+        max_l2_samples=2000,
+        whisper_model="openai/whisper-tiny",
+        loss_fn="focal",  # or "bce" for Pipecat's original
+        fp_penalty=2.0,   # interrupting learner costs 2x more than waiting
+    )

03-finetune-pipecat-pt/05_evaluate.py

@@ -0,0 +1,511 @@
+"""Evaluate fine-tuned model against Pipecat baseline and ONNX reference.
+
+Comparisons:
+1. Our fine-tuned model (PyTorch) on Pipecat PT test set
+2. Pipecat original ONNX model on same test set
+3. Threshold sweep to find optimal operating point
+4. Per-source breakdown (pipecat data vs our TTS data)
+
+Run:
+    python 05_evaluate.py --model results/best_model.pt
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import time
+from pathlib import Path
+
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+
+log = logging.getLogger(__name__)
+
+DATA_DIR = Path(__file__).parent / "data"
+_workspace = Path("/workspace") if Path("/workspace").exists() else Path(".")
+RESULTS_DIR = _workspace / "results"
+CACHE_DIR = _workspace / "hf_cache"
+
+SAMPLE_RATE = 16000
+WINDOW_SECONDS = 8
+WINDOW_SAMPLES = WINDOW_SECONDS * SAMPLE_RATE
+
+# Pipecat baseline (from their blog: Smart Turn v3 Portuguese)
+PIPECAT_BASELINE = {
+    "accuracy": 0.9542,
+    "fp_rate": 0.0279,
+    "fn_rate": 0.0179,
+    "source": "https://www.daily.co/blog/announcing-smart-turn-v3-with-cpu-inference-in-just-12ms/",
+}
+
+
+def load_model(model_path: Path, device: str) -> nn.Module:
+    """Load fine-tuned SmartTurnV3Model."""
+    # Import from finetune script
+    import importlib.util
+    spec = importlib.util.spec_from_file_location(
+        "finetune", Path(__file__).parent / "04_finetune.py"
+    )
+    finetune = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(finetune)
+
+    model = finetune.SmartTurnV3Model(whisper_model="openai/whisper-tiny")
+    checkpoint = torch.load(model_path, map_location=device, weights_only=True)
+    model.load_state_dict(checkpoint["model_state_dict"])
+    model = model.to(device)
+    model.eval()
+
+    log.info("Loaded model from %s (epoch %d, val_f1=%.4f)",
+             model_path, checkpoint.get("epoch", -1), checkpoint.get("val_f1", -1))
+    return model
+
+
+def load_onnx_model(onnx_dir: Path | None = None):
+    """Load Pipecat ONNX model for comparison."""
+    try:
+        import onnxruntime as ort
+    except ImportError:
+        log.warning("onnxruntime not installed — skipping ONNX evaluation")
+        return None
+
+    if onnx_dir is None:
+        onnx_dir = DATA_DIR / "pipecat_model"
+
+    cpu_path = onnx_dir / "smart-turn-v3.2-cpu.onnx"
+    gpu_path = onnx_dir / "smart-turn-v3.2-gpu.onnx"
+
+    onnx_path = cpu_path if cpu_path.exists() else (gpu_path if gpu_path.exists() else None)
+    if onnx_path is None:
+        log.warning("No Pipecat ONNX model found in %s", onnx_dir)
+        return None
+
+    session = ort.InferenceSession(str(onnx_path))
+    log.info("Loaded ONNX model: %s", onnx_path)
+    return session
+
+
+def load_test_data() -> tuple[list, list]:
+    """Load test datasets: Pipecat PT test + our TTS test."""
+    import soundfile as sf
+    from transformers import WhisperFeatureExtractor
+
+    feature_extractor = WhisperFeatureExtractor(chunk_length=8)
+
+    # 1. Pipecat test data
+    pipecat_samples = []
+    test_dir = DATA_DIR / "pipecat_pt_test"
+    if test_dir.exists():
+        for wav_path in sorted(test_dir.glob("*.wav")):
+            try:
+                audio, sr = sf.read(str(wav_path))
+                audio = np.array(audio, dtype=np.float32)
+                if sr != SAMPLE_RATE:
+                    import torchaudio
+                    tensor = torch.from_numpy(audio).float().unsqueeze(0)
+                    audio = torchaudio.functional.resample(tensor, sr, SAMPLE_RATE).squeeze().numpy()
+
+                audio = _extract_window(audio)
+                if audio is not None:
+                    label = 1.0 if "_complete" in wav_path.name else 0.0
+                    features = feature_extractor(
+                        audio, sampling_rate=SAMPLE_RATE,
+                        return_tensors="np", padding="max_length",
+                        max_length=WINDOW_SAMPLES, truncation=True,
+                        do_normalize=True,
+                    ).input_features.squeeze(0).astype(np.float32)
+
+                    pipecat_samples.append({
+                        "features": features,
+                        "label": label,
+                        "source": "pipecat_test",
+                        "file": wav_path.name,
+                    })
+            except Exception as e:
+                pass
+
+    log.info("Pipecat test: %d samples", len(pipecat_samples))
+
+    # 2. Our TTS test data
+    tts_samples = []
+    tts_dir = DATA_DIR / "tts_dataset"
+    meta_path = tts_dir / "metadata.json"
+    if meta_path.exists():
+        with open(meta_path) as f:
+            metadata = json.load(f)
+
+        # Use last 20% as test
+        test_start = int(len(metadata) * 0.8)
+        test_meta = metadata[test_start:]
+
+        audio_dir = tts_dir / "audio"
+        for meta in test_meta:
+            wav_path = audio_dir / meta["file"]
+            if not wav_path.exists():
+                continue
+            try:
+                audio, sr = sf.read(str(wav_path))
+                audio = np.array(audio, dtype=np.float32)
+                if sr != SAMPLE_RATE:
+                    import torchaudio
+                    tensor = torch.from_numpy(audio).float().unsqueeze(0)
+                    audio = torchaudio.functional.resample(tensor, sr, SAMPLE_RATE).squeeze().numpy()
+
+                audio = _extract_window(audio)
+                if audio is not None:
+                    label = 1.0 if meta["label"] == "complete" else 0.0
+                    features = feature_extractor(
+                        audio, sampling_rate=SAMPLE_RATE,
+                        return_tensors="np", padding="max_length",
+                        max_length=WINDOW_SAMPLES, truncation=True,
+                        do_normalize=True,
+                    ).input_features.squeeze(0).astype(np.float32)
+
+                    tts_samples.append({
+                        "features": features,
+                        "label": label,
+                        "source": meta.get("accent", "unknown"),
+                        "file": meta["file"],
+                    })
+            except Exception:
+                pass
+
+    log.info("TTS test: %d samples", len(tts_samples))
+    return pipecat_samples, tts_samples
+
+
+def _extract_window(audio: np.ndarray) -> np.ndarray | None:
+    """Extract 8s window from end of audio."""
+    if len(audio) < SAMPLE_RATE:
+        return None
+    peak = np.max(np.abs(audio))
+    if peak > 0:
+        audio = audio / peak * 0.9
+    if len(audio) > WINDOW_SAMPLES:
+        audio = audio[-WINDOW_SAMPLES:]
+    elif len(audio) < WINDOW_SAMPLES:
+        audio = np.pad(audio, (WINDOW_SAMPLES - len(audio), 0), mode="constant")
+    audio[-int(0.2 * SAMPLE_RATE):] = 0.0
+    return audio.astype(np.float32)
+
+
+# ---------------------------------------------------------------------------
+# Evaluation functions
+# ---------------------------------------------------------------------------
+
+def evaluate_pytorch(
+    model: nn.Module,
+    samples: list[dict],
+    device: str,
+    threshold: float = 0.5,
+) -> dict:
+    """Evaluate PyTorch model on samples."""
+    if not samples:
+        return {}
+
+    tp = fp = fn = tn = 0
+    latencies = []
+
+    with torch.no_grad():
+        for s in samples:
+            features = torch.from_numpy(s["features"]).unsqueeze(0).to(device)
+
+            t0 = time.perf_counter()
+            logits = model(features)
+            latency_ms = (time.perf_counter() - t0) * 1000
+            latencies.append(latency_ms)
+
+            pred = float(torch.sigmoid(logits).item() > threshold)
+            label = s["label"]
+
+            if pred == 1 and label == 1:
+                tp += 1
+            elif pred == 1 and label == 0:
+                fp += 1
+            elif pred == 0 and label == 1:
+                fn += 1
+            else:
+                tn += 1
+
+    total = tp + fp + fn + tn
+    return _compute_metrics(tp, fp, fn, tn, latencies)
+
+
+def evaluate_onnx(
+    session,
+    samples: list[dict],
+    threshold: float = 0.5,
+) -> dict:
+    """Evaluate ONNX model on samples."""
+    if not samples or session is None:
+        return {}
+
+    tp = fp = fn = tn = 0
+    latencies = []
+
+    input_name = session.get_inputs()[0].name
+    for s in samples:
+        features = s["features"][np.newaxis, ...]
+
+        t0 = time.perf_counter()
+        outputs = session.run(None, {input_name: features})
+        latency_ms = (time.perf_counter() - t0) * 1000
+        latencies.append(latency_ms)
+
+        logit = outputs[0].item() if outputs[0].size == 1 else outputs[0][0].item()
+        pred = float(1.0 / (1.0 + np.exp(-logit)) > threshold)
+        label = s["label"]
+
+        if pred == 1 and label == 1:
+            tp += 1
+        elif pred == 1 and label == 0:
+            fp += 1
+        elif pred == 0 and label == 1:
+            fn += 1
+        else:
+            tn += 1
+
+    return _compute_metrics(tp, fp, fn, tn, latencies)
+
+
+def _compute_metrics(tp, fp, fn, tn, latencies=None) -> dict:
+    total = tp + fp + fn + tn
+    accuracy = (tp + tn) / max(total, 1)
+    precision = tp / max(tp + fp, 1)
+    recall = tp / max(tp + fn, 1)
+    f1 = 2 * precision * recall / max(precision + recall, 1e-8)
+    fp_rate = fp / max(fp + tn, 1)
+    fn_rate = fn / max(fn + tp, 1)
+
+    metrics = {
+        "accuracy": round(accuracy, 4),
+        "precision": round(precision, 4),
+        "recall": round(recall, 4),
+        "f1": round(f1, 4),
+        "fp_rate": round(fp_rate, 4),
+        "fn_rate": round(fn_rate, 4),
+        "tp": tp, "fp": fp, "fn": fn, "tn": tn,
+        "total": total,
+    }
+
+    if latencies:
+        metrics["latency_mean_ms"] = round(np.mean(latencies), 2)
+        metrics["latency_p50_ms"] = round(np.median(latencies), 2)
+        metrics["latency_p95_ms"] = round(np.percentile(latencies, 95), 2)
+
+    return metrics
+
+
+# ---------------------------------------------------------------------------
+# Main evaluation
+# ---------------------------------------------------------------------------
+
+def run_evaluation(model_path: Path) -> dict:
+    """Run full evaluation suite."""
+    device = "cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")
+    log.info("Evaluating on device: %s", device)
+
+    # Load models
+    model = load_model(model_path, device)
+    onnx_session = load_onnx_model()
+
+    # Load test data
+    pipecat_samples, tts_samples = load_test_data()
+    all_samples = pipecat_samples + tts_samples
+
+    results = {"pipecat_baseline": PIPECAT_BASELINE}
+
+    # 1. Our model on all test data
+    log.info("\n=== Our model (all test data) ===")
+    our_all = evaluate_pytorch(model, all_samples, device)
+    if our_all:
+        _log_metrics("Our model (all)", our_all)
+        results["our_model_all"] = our_all
+
+    # 2. Our model on Pipecat test only
+    if pipecat_samples:
+        log.info("\n=== Our model (Pipecat PT test only) ===")
+        our_pipecat = evaluate_pytorch(model, pipecat_samples, device)
+        _log_metrics("Our model (Pipecat test)", our_pipecat)
+        results["our_model_pipecat_test"] = our_pipecat
+
+        # Compare with baseline
+        diff_acc = our_pipecat["accuracy"] - PIPECAT_BASELINE["accuracy"]
+        diff_fp = our_pipecat["fp_rate"] - PIPECAT_BASELINE["fp_rate"]
+        diff_fn = our_pipecat["fn_rate"] - PIPECAT_BASELINE["fn_rate"]
+        log.info("  vs Pipecat baseline: accuracy %+.2f%%, FP %+.2f%%, FN %+.2f%%",
+                 diff_acc * 100, diff_fp * 100, diff_fn * 100)
+
+    # 3. Our model per accent
+    for accent_name, accent_filter in [("native_pt_br", "native_pt_br"), ("french_pt", "french_pt")]:
+        accent_samples = [s for s in tts_samples if s["source"] == accent_filter]
+        if accent_samples:
+            log.info("\n=== Our model (%s) ===", accent_name)
+            m = evaluate_pytorch(model, accent_samples, device)
+            _log_metrics(f"Our model ({accent_name})", m)
+            results[f"our_model_{accent_name}"] = m
+
+    # 4. ONNX reference model
+    if onnx_session and pipecat_samples:
+        log.info("\n=== Pipecat ONNX model (reference) ===")
+        onnx_metrics = evaluate_onnx(onnx_session, pipecat_samples)
+        if onnx_metrics:
+            _log_metrics("Pipecat ONNX", onnx_metrics)
+            results["pipecat_onnx"] = onnx_metrics
+
+    # 5. Threshold sweep
+    log.info("\n=== Threshold Sweep (all test data) ===")
+    sweep = {}
+    for thresh in [0.3, 0.4, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9]:
+        m = evaluate_pytorch(model, all_samples, device, threshold=thresh)
+        sweep[str(thresh)] = m
+        log.info("  threshold=%.2f: acc=%.3f prec=%.3f rec=%.3f f1=%.3f FP=%.3f FN=%.3f",
+                 thresh, m["accuracy"], m["precision"], m["recall"],
+                 m["f1"], m["fp_rate"], m["fn_rate"])
+
+    results["threshold_sweep"] = sweep
+
+    # Find optimal thresholds
+    best_f1_thresh = max(sweep.items(), key=lambda x: x[1]["f1"])
+    best_prec_thresh = max(
+        [(k, v) for k, v in sweep.items() if v["recall"] >= 0.90],
+        key=lambda x: x[1]["precision"],
+        default=(None, None),
+    )
+
+    log.info("\n=== Recommendations ===")
+    log.info("  Best F1: threshold=%.2f (F1=%.3f, prec=%.3f, rec=%.3f)",
+             float(best_f1_thresh[0]), best_f1_thresh[1]["f1"],
+             best_f1_thresh[1]["precision"], best_f1_thresh[1]["recall"])
+    if best_prec_thresh[0]:
+        log.info("  Best precision (recall>=90%%): threshold=%.2f (prec=%.3f, rec=%.3f)",
+                 float(best_prec_thresh[0]), best_prec_thresh[1]["precision"],
+                 best_prec_thresh[1]["recall"])
+
+    # ----- Dual Threshold for Language Learning Avatar -----
+    # Inspired by Deepgram Flux (eot_threshold + eager_eot_threshold)
+    # For L2 learners: higher final threshold to avoid interrupting mid-hesitation
+    log.info("\n=== Dual Threshold (Language Learning Mode) ===")
+    log.info("  Context: conversational avatar for francophones learning Portuguese")
+    log.info("  Priority: minimize interruptions (FP) over fast response (FN)")
+
+    # Final threshold: minimize FP rate while keeping recall >= 85%
+    final_candidates = [(k, v) for k, v in sweep.items()
+                        if v["recall"] >= 0.85 and float(k) >= 0.6]
+    if final_candidates:
+        best_final = min(final_candidates, key=lambda x: x[1]["fp_rate"])
+        final_thresh = float(best_final[0])
+        log.info("  Final threshold: %.2f (FP=%.1f%%, rec=%.1f%%)",
+                 final_thresh, best_final[1]["fp_rate"] * 100,
+                 best_final[1]["recall"] * 100)
+    else:
+        final_thresh = 0.7
+        log.info("  Final threshold: 0.70 (default)")
+
+    # Eager threshold: lower confidence for speculative LLM generation
+    eager_thresh = max(0.3, final_thresh - 0.3)
+    log.info("  Eager threshold: %.2f (start speculative LLM prep)", eager_thresh)
+    log.info("  Expected latency savings: ~150-250ms on response start")
+
+    # Simulate dual-threshold behavior
+    if all_samples:
+        log.info("\n  Dual-threshold simulation:")
+        eager_m = evaluate_pytorch(model, all_samples, device, threshold=eager_thresh)
+        final_m = evaluate_pytorch(model, all_samples, device, threshold=final_thresh)
+        log.info("    Eager (%.2f): would trigger on %.1f%% of samples (%.1f%% false triggers)",
+                 eager_thresh, (eager_m["tp"] + eager_m["fp"]) / max(eager_m["total"], 1) * 100,
+                 eager_m["fp_rate"] * 100)
+        log.info("    Final (%.2f): confirms %.1f%% of turns (%.1f%% false confirms, %.1f%% missed)",
+                 final_thresh, final_m["recall"] * 100,
+                 final_m["fp_rate"] * 100, final_m["fn_rate"] * 100)
+        wasted_speculative = eager_m["fp"] - final_m["fp"]
+        log.info("    Wasted speculative preps: %d (started but not confirmed)", max(0, wasted_speculative))
+
+    results["recommended"] = {
+        "best_f1_threshold": float(best_f1_thresh[0]),
+        "best_precision_threshold": float(best_prec_thresh[0]) if best_prec_thresh[0] else None,
+        "dual_threshold": {
+            "eager": eager_thresh,
+            "final": final_thresh,
+            "mode": "language_learning",
+            "rationale": "Higher final threshold minimizes interruptions for L2 learners "
+                        "who pause 500-2000ms mid-sentence. Eager threshold enables "
+                        "speculative LLM prep for lower perceived latency.",
+        },
+    }
+
+    # 6. Summary comparison table
+    log.info("\n" + "=" * 70)
+    log.info("SUMMARY COMPARISON")
+    log.info("=" * 70)
+    log.info("%-30s %8s %8s %8s %8s", "Model", "Acc", "FP%", "FN%", "F1")
+    log.info("-" * 70)
+    log.info("%-30s %7.1f%% %7.2f%% %7.2f%% %8s",
+             "Pipecat v3.2 (baseline)",
+             PIPECAT_BASELINE["accuracy"] * 100,
+             PIPECAT_BASELINE["fp_rate"] * 100,
+             PIPECAT_BASELINE["fn_rate"] * 100,
+             "~0.96")
+
+    if "pipecat_onnx" in results:
+        m = results["pipecat_onnx"]
+        log.info("%-30s %7.1f%% %7.2f%% %7.2f%% %7.3f",
+                 "Pipecat ONNX (our eval)",
+                 m["accuracy"] * 100, m["fp_rate"] * 100,
+                 m["fn_rate"] * 100, m["f1"])
+
+    if "our_model_pipecat_test" in results:
+        m = results["our_model_pipecat_test"]
+        log.info("%-30s %7.1f%% %7.2f%% %7.2f%% %7.3f",
+                 "Our model (Pipecat test)",
+                 m["accuracy"] * 100, m["fp_rate"] * 100,
+                 m["fn_rate"] * 100, m["f1"])
+
+    if "our_model_all" in results:
+        m = results["our_model_all"]
+        log.info("%-30s %7.1f%% %7.2f%% %7.2f%% %7.3f",
+                 "Our model (all test)",
+                 m["accuracy"] * 100, m["fp_rate"] * 100,
+                 m["fn_rate"] * 100, m["f1"])
+
+    log.info("=" * 70)
+
+    # Save results
+    RESULTS_DIR.mkdir(parents=True, exist_ok=True)
+    results_path = RESULTS_DIR / "evaluation_results.json"
+    with open(results_path, "w") as f:
+        json.dump(results, f, indent=2)
+    log.info("\nResults saved to %s", results_path)
+
+    return results
+
+
+def _log_metrics(name: str, m: dict) -> None:
+    log.info("%s:", name)
+    log.info("  Accuracy:  %.3f (%.1f%%)", m["accuracy"], m["accuracy"] * 100)
+    log.info("  Precision: %.3f", m["precision"])
+    log.info("  Recall:    %.3f", m["recall"])
+    log.info("  F1:        %.3f", m["f1"])
+    log.info("  FP rate:   %.3f (%.1f%%)", m["fp_rate"], m["fp_rate"] * 100)
+    log.info("  FN rate:   %.3f (%.1f%%)", m["fn_rate"], m["fn_rate"] * 100)
+    log.info("  TP=%d FP=%d FN=%d TN=%d (total=%d)", m["tp"], m["fp"], m["fn"], m["tn"], m["total"])
+    if "latency_mean_ms" in m:
+        log.info("  Latency: mean=%.1fms p50=%.1fms p95=%.1fms",
+                 m["latency_mean_ms"], m["latency_p50_ms"], m["latency_p95_ms"])
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s [%(name)s] %(message)s",
+    )
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", type=str, default=str(RESULTS_DIR / "best_model.pt"),
+                        help="Path to fine-tuned model checkpoint")
+    args = parser.parse_args()
+
+    run_evaluation(Path(args.model))

03-finetune-pipecat-pt/06_inference.py

@@ -0,0 +1,560 @@
+"""Inference engine for the language-learning avatar turn-taking model.
+
+This module implements the dual-threshold + backchannel system designed
+for a conversational avatar that teaches Portuguese to French speakers.
+
+Key insight: L2 learners pause 1-3s mid-sentence (word search, conjugation,
+code-switching). A naive system either:
+  a) Interrupts them (bad UX — kills confidence), or
+  b) Stays silent too long (learner thinks avatar froze)
+
+Solution: dual-threshold with backchannel signals.
+
+Architecture:
+    Audio stream → SmartTurnV3 model → confidence score (0.0-1.0)
+        │
+        ├─ score < eager_threshold (0.4)  → LISTENING (do nothing)
+        ├─ eager ≤ score < final (0.7)    → PREPARING (backchannel + speculative LLM)
+        └─ score ≥ final_threshold (0.7)  → RESPONDING (take the turn)
+
+    Silence timer (parallel):
+        0-600ms    → normal (no action)
+        600ms-1.5s → visual backchannel (nod, eye contact)
+        1.5s-3.0s  → verbal backchannel ("mhm", "continue...")
+        3.0s+      → encouragement ("sem pressa, pode pensar...")
+
+References:
+- Deepgram Flux: eot_threshold + eager_eot_threshold (2025)
+- ConversAR: "infinite thinking period" for L2 learners (2025)
+- Tavus: 600ms response latency threshold (2025)
+- Kosmala (2022): L2 repair pauses average 844ms
+- Cenoz (2000): L2 silent pauses range 205ms to 11,569ms
+
+Usage:
+    engine = TurnTakingEngine(model_path="results/smart_turn_pt_v3.onnx")
+    engine.on_state_change(my_callback)
+
+    # Feed audio chunks continuously
+    for chunk in audio_stream:
+        engine.feed_audio(chunk)
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from pathlib import Path
+from typing import Callable
+
+import numpy as np
+
+log = logging.getLogger(__name__)
+
+SAMPLE_RATE = 16000
+WINDOW_SECONDS = 8
+WINDOW_SAMPLES = WINDOW_SECONDS * SAMPLE_RATE
+
+
+# ---------------------------------------------------------------------------
+# States and events
+# ---------------------------------------------------------------------------
+
+class TurnState(Enum):
+    """Avatar turn-taking state machine."""
+    LISTENING = "listening"           # User is speaking, avatar listens
+    SILENCE = "silence"               # User stopped, waiting to classify
+    BACKCHANNEL_VISUAL = "bc_visual"  # Nod/eye contact (600ms-1.5s silence)
+    BACKCHANNEL_VERBAL = "bc_verbal"  # "Mhm" / "continue" (1.5s-3.0s)
+    ENCOURAGEMENT = "encouragement"   # "Sem pressa..." (3.0s+ silence)
+    PREPARING = "preparing"           # Eager threshold hit, speculative LLM started
+    RESPONDING = "responding"         # Final threshold hit, avatar speaks
+
+
+@dataclass
+class TurnEvent:
+    """Event emitted on state transitions."""
+    state: TurnState
+    confidence: float          # Model's end-of-turn confidence (0-1)
+    silence_duration_ms: float # How long the user has been silent
+    timestamp: float           # time.monotonic()
+    message: str = ""          # Backchannel text (if applicable)
+
+
+@dataclass
+class BackchannelConfig:
+    """Configurable backchannel behavior for the avatar.
+
+    These can be tuned per learner profile or CEFR level:
+    - A1/A2: longer delays, more encouraging messages
+    - B1/B2: shorter delays, less frequent backchannels
+    """
+    # Silence thresholds (milliseconds)
+    visual_backchannel_ms: float = 600.0
+    verbal_backchannel_ms: float = 1500.0
+    encouragement_ms: float = 3000.0
+
+    # Model confidence thresholds
+    eager_threshold: float = 0.4    # Start speculative LLM generation
+    final_threshold: float = 0.7    # Confirm end-of-turn, avatar speaks
+
+    # Backchannel messages (Portuguese, with French-speaker-friendly variants)
+    visual_actions: list[str] = field(default_factory=lambda: [
+        "nod",          # Aceno de cabeça
+        "eye_contact",  # Olhar atento
+        "slight_smile", # Sorriso leve
+    ])
+
+    verbal_backchannels: list[str] = field(default_factory=lambda: [
+        "Mhm...",
+        "Uhum...",
+        "Sim...",
+        "Tá...",
+        "Sei...",
+        "Continue...",
+    ])
+
+    encouragement_messages: list[str] = field(default_factory=lambda: [
+        "Pode continuar, sem pressa...",
+        "Tá pensando? Tranquilo...",
+        "Pode pensar, eu espero...",
+        "Sem pressa, tá tudo bem...",
+        "Take your time... pode falar em português...",  # Code-switch friendly
+        "Prenez votre temps... quando estiver pronto...",  # French reassurance
+    ])
+
+    # Cooldowns (don't spam backchannels)
+    verbal_cooldown_ms: float = 3000.0   # Min time between verbal backchannels
+    encouragement_cooldown_ms: float = 8000.0  # Min time between encouragements
+
+
+# ---------------------------------------------------------------------------
+# CEFR-aware presets
+# ---------------------------------------------------------------------------
+
+CEFR_PRESETS: dict[str, BackchannelConfig] = {
+    "A1": BackchannelConfig(
+        visual_backchannel_ms=500.0,
+        verbal_backchannel_ms=1200.0,
+        encouragement_ms=2500.0,
+        eager_threshold=0.5,     # Higher — need more confidence before preparing
+        final_threshold=0.8,     # Much higher — very patient, rarely interrupt
+    ),
+    "A2": BackchannelConfig(
+        visual_backchannel_ms=550.0,
+        verbal_backchannel_ms=1300.0,
+        encouragement_ms=2800.0,
+        eager_threshold=0.45,
+        final_threshold=0.75,
+    ),
+    "B1": BackchannelConfig(
+        visual_backchannel_ms=600.0,
+        verbal_backchannel_ms=1500.0,
+        encouragement_ms=3000.0,
+        eager_threshold=0.4,
+        final_threshold=0.7,     # Default
+    ),
+    "B2": BackchannelConfig(
+        visual_backchannel_ms=600.0,
+        verbal_backchannel_ms=1800.0,
+        encouragement_ms=4000.0,
+        eager_threshold=0.35,
+        final_threshold=0.65,    # More responsive — B2 pauses less
+    ),
+    "C1": BackchannelConfig(
+        visual_backchannel_ms=600.0,
+        verbal_backchannel_ms=2000.0,
+        encouragement_ms=5000.0,
+        eager_threshold=0.35,
+        final_threshold=0.6,     # Near-native responsiveness
+    ),
+}
+
+
+# ---------------------------------------------------------------------------
+# Turn-taking engine
+# ---------------------------------------------------------------------------
+
+class TurnTakingEngine:
+    """Real-time turn-taking engine for language-learning avatar.
+
+    Combines the SmartTurnV3 model with a silence timer and backchannel
+    state machine to create a patient, encouraging conversational partner.
+
+    Usage:
+        engine = TurnTakingEngine("results/smart_turn_pt_v3.onnx")
+        engine.on_state_change(handle_event)
+
+        # In audio loop:
+        for chunk in mic_stream:
+            engine.feed_audio(chunk)
+
+        # Or feed pre-computed features:
+        engine.feed_score(model_confidence, is_speech=True)
+    """
+
+    def __init__(
+        self,
+        model_path: str | Path | None = None,
+        config: BackchannelConfig | None = None,
+        cefr_level: str = "B1",
+    ):
+        # Config: use CEFR preset or custom
+        if config is not None:
+            self.config = config
+        elif cefr_level in CEFR_PRESETS:
+            self.config = CEFR_PRESETS[cefr_level]
+            log.info("Using CEFR %s preset: eager=%.2f, final=%.2f",
+                     cefr_level, self.config.eager_threshold, self.config.final_threshold)
+        else:
+            self.config = BackchannelConfig()
+
+        # State
+        self._state = TurnState.LISTENING
+        self._silence_start: float | None = None
+        self._last_speech_time: float = time.monotonic()
+        self._last_verbal_bc: float = 0.0
+        self._last_encouragement: float = 0.0
+        self._last_confidence: float = 0.0
+        self._callbacks: list[Callable[[TurnEvent], None]] = []
+        self._audio_buffer = np.zeros(WINDOW_SAMPLES, dtype=np.float32)
+        self._speculative_started = False
+
+        # Load ONNX model if provided
+        self._session = None
+        self._feature_extractor = None
+        if model_path is not None:
+            self._load_model(model_path)
+
+    def _load_model(self, model_path: str | Path) -> None:
+        """Load ONNX model for inference."""
+        try:
+            import onnxruntime as ort
+            self._session = ort.InferenceSession(
+                str(model_path),
+                providers=["CPUExecutionProvider"],
+            )
+            log.info("Loaded turn-taking model: %s", model_path)
+        except Exception as e:
+            log.warning("Failed to load model %s: %s — running in manual mode", model_path, e)
+
+        try:
+            from transformers import WhisperFeatureExtractor
+            self._feature_extractor = WhisperFeatureExtractor(chunk_length=WINDOW_SECONDS)
+        except ImportError:
+            log.warning("transformers not available — cannot extract features")
+
+    def on_state_change(self, callback: Callable[[TurnEvent], None]) -> None:
+        """Register callback for state change events."""
+        self._callbacks.append(callback)
+
+    @property
+    def state(self) -> TurnState:
+        return self._state
+
+    @property
+    def silence_duration_ms(self) -> float:
+        if self._silence_start is None:
+            return 0.0
+        return (time.monotonic() - self._silence_start) * 1000
+
+    # ----- Audio input -----
+
+    def feed_audio(self, chunk: np.ndarray, is_speech: bool | None = None) -> TurnEvent | None:
+        """Feed an audio chunk and get turn-taking decision.
+
+        chunk: float32 audio at 16kHz
+        is_speech: if None, will use simple energy-based VAD
+
+        Returns TurnEvent if state changed, None otherwise.
+        """
+        now = time.monotonic()
+
+        # Update audio buffer (sliding window)
+        chunk = chunk.astype(np.float32)
+        if len(chunk) >= WINDOW_SAMPLES:
+            self._audio_buffer = chunk[-WINDOW_SAMPLES:]
+        else:
+            self._audio_buffer = np.roll(self._audio_buffer, -len(chunk))
+            self._audio_buffer[-len(chunk):] = chunk
+
+        # Simple VAD if not provided
+        if is_speech is None:
+            rms = np.sqrt(np.mean(chunk ** 2))
+            is_speech = rms > 0.01  # Simple threshold
+
+        if is_speech:
+            return self._on_speech(now)
+        else:
+            return self._on_silence(now)
+
+    def feed_score(self, confidence: float, is_speech: bool) -> TurnEvent | None:
+        """Feed pre-computed model confidence score.
+
+        Use this when you run the model externally and just want
+        the backchannel/threshold logic.
+
+        confidence: 0.0 (definitely incomplete) to 1.0 (definitely complete)
+        is_speech: whether VAD detected speech in this frame
+        """
+        now = time.monotonic()
+        self._last_confidence = confidence
+
+        if is_speech:
+            return self._on_speech(now)
+        else:
+            return self._on_silence(now, confidence=confidence)
+
+    # ----- Internal state machine -----
+
+    def _on_speech(self, now: float) -> TurnEvent | None:
+        """User is speaking."""
+        self._last_speech_time = now
+        self._silence_start = None
+        self._speculative_started = False
+
+        if self._state != TurnState.LISTENING:
+            return self._transition(TurnState.LISTENING, confidence=0.0, now=now)
+        return None
+
+    def _on_silence(self, now: float, confidence: float | None = None) -> TurnEvent | None:
+        """User is silent — run the state machine."""
+        # Mark silence start
+        if self._silence_start is None:
+            self._silence_start = now
+
+        silence_ms = (now - self._silence_start) * 1000
+
+        # Get model confidence if we have a model and no external score
+        if confidence is None:
+            confidence = self._run_model()
+        self._last_confidence = confidence
+
+        cfg = self.config
+
+        # ----- Decision tree -----
+
+        # 1. Final threshold → RESPONDING (take the turn)
+        if confidence >= cfg.final_threshold:
+            return self._transition(TurnState.RESPONDING, confidence, now)
+
+        # 2. Eager threshold → PREPARING (speculative LLM, backchannel)
+        if confidence >= cfg.eager_threshold and not self._speculative_started:
+            self._speculative_started = True
+            return self._transition(TurnState.PREPARING, confidence, now)
+
+        # 3. Silence-based backchannels (even if model is unsure)
+        # These keep the learner engaged so they don't think the avatar froze
+
+        if silence_ms >= cfg.encouragement_ms:
+            if now - self._last_encouragement >= cfg.encouragement_cooldown_ms / 1000:
+                self._last_encouragement = now
+                return self._transition(TurnState.ENCOURAGEMENT, confidence, now)
+
+        if silence_ms >= cfg.verbal_backchannel_ms:
+            if now - self._last_verbal_bc >= cfg.verbal_cooldown_ms / 1000:
+                self._last_verbal_bc = now
+                return self._transition(TurnState.BACKCHANNEL_VERBAL, confidence, now)
+
+        if silence_ms >= cfg.visual_backchannel_ms:
+            if self._state not in (TurnState.BACKCHANNEL_VISUAL,
+                                    TurnState.BACKCHANNEL_VERBAL,
+                                    TurnState.ENCOURAGEMENT,
+                                    TurnState.PREPARING):
+                return self._transition(TurnState.BACKCHANNEL_VISUAL, confidence, now)
+
+        # Still in silence, no state change
+        if self._state == TurnState.LISTENING:
+            return self._transition(TurnState.SILENCE, confidence, now)
+
+        return None
+
+    def _transition(self, new_state: TurnState, confidence: float, now: float) -> TurnEvent:
+        """Transition to a new state and emit event."""
+        import random
+
+        old_state = self._state
+        self._state = new_state
+
+        silence_ms = (now - self._silence_start) * 1000 if self._silence_start else 0.0
+
+        # Pick appropriate message
+        message = ""
+        if new_state == TurnState.BACKCHANNEL_VISUAL:
+            message = random.choice(self.config.visual_actions)
+        elif new_state == TurnState.BACKCHANNEL_VERBAL:
+            message = random.choice(self.config.verbal_backchannels)
+        elif new_state == TurnState.ENCOURAGEMENT:
+            message = random.choice(self.config.encouragement_messages)
+        elif new_state == TurnState.PREPARING:
+            message = "speculative_llm_start"
+        elif new_state == TurnState.RESPONDING:
+            message = "take_turn"
+
+        event = TurnEvent(
+            state=new_state,
+            confidence=confidence,
+            silence_duration_ms=silence_ms,
+            timestamp=now,
+            message=message,
+        )
+
+        if old_state != new_state:
+            log.debug("Turn state: %s → %s (conf=%.2f, silence=%.0fms, msg=%s)",
+                      old_state.value, new_state.value, confidence, silence_ms, message)
+
+        for cb in self._callbacks:
+            try:
+                cb(event)
+            except Exception as e:
+                log.warning("Callback error: %s", e)
+
+        return event
+
+    def _run_model(self) -> float:
+        """Run the ONNX model on current audio buffer."""
+        if self._session is None or self._feature_extractor is None:
+            return 0.0
+
+        try:
+            inputs = self._feature_extractor(
+                self._audio_buffer,
+                sampling_rate=SAMPLE_RATE,
+                return_tensors="np",
+                padding="max_length",
+                max_length=WINDOW_SAMPLES,
+                truncation=True,
+                do_normalize=True,
+            )
+            features = inputs.input_features.astype(np.float32)
+
+            input_name = self._session.get_inputs()[0].name
+            outputs = self._session.run(None, {input_name: features})
+            logit = outputs[0].item() if outputs[0].size == 1 else outputs[0][0].item()
+
+            # Sigmoid
+            confidence = 1.0 / (1.0 + np.exp(-logit))
+            return float(confidence)
+        except Exception as e:
+            log.warning("Model inference error: %s", e)
+            return 0.0
+
+    # ----- Convenience -----
+
+    def reset(self) -> None:
+        """Reset state (e.g., when starting a new conversation turn)."""
+        self._state = TurnState.LISTENING
+        self._silence_start = None
+        self._speculative_started = False
+        self._last_confidence = 0.0
+        self._audio_buffer = np.zeros(WINDOW_SAMPLES, dtype=np.float32)
+
+    def set_cefr_level(self, level: str) -> None:
+        """Change CEFR level at runtime (e.g., after proficiency assessment)."""
+        if level in CEFR_PRESETS:
+            self.config = CEFR_PRESETS[level]
+            log.info("Switched to CEFR %s: eager=%.2f, final=%.2f",
+                     level, self.config.eager_threshold, self.config.final_threshold)
+        else:
+            log.warning("Unknown CEFR level: %s", level)
+
+
+# ---------------------------------------------------------------------------
+# Example usage and demo
+# ---------------------------------------------------------------------------
+
+def demo_simulation():
+    """Simulate a conversation to demonstrate the backchannel system.
+
+    Scenario: French speaker (B1) learning Portuguese, pausing frequently.
+    """
+    print("=" * 70)
+    print("DEMO: Turn-Taking Engine for Language Learning Avatar")
+    print("Scenario: French B1 learner speaking Portuguese")
+    print("=" * 70)
+
+    events_log = []
+
+    def on_event(event: TurnEvent):
+        events_log.append(event)
+        state_emoji = {
+            TurnState.LISTENING: "👂",
+            TurnState.SILENCE: "⏸️",
+            TurnState.BACKCHANNEL_VISUAL: "😊",
+            TurnState.BACKCHANNEL_VERBAL: "💬",
+            TurnState.ENCOURAGEMENT: "🤗",
+            TurnState.PREPARING: "🧠",
+            TurnState.RESPONDING: "🗣️",
+        }
+        emoji = state_emoji.get(event.state, "❓")
+        print(f"  {emoji} [{event.silence_duration_ms:6.0f}ms] "
+              f"conf={event.confidence:.2f} → {event.state.value}: {event.message}")
+
+    engine = TurnTakingEngine(config=CEFR_PRESETS["B1"])
+    engine.on_state_change(on_event)
+
+    # Simulate a conversation timeline
+    # Each entry: (description, duration_ms, is_speech, model_confidence)
+    timeline = [
+        # Learner starts speaking
+        ("Learner: 'Eu fui ao...'", 1200, True, 0.1),
+        # Pause — searching for word (conjugation hesitation)
+        ("  [silence — thinking about conjugation]", 300, False, 0.15),
+        ("  [still thinking...]", 400, False, 0.20),
+        ("  [600ms — visual backchannel]", 300, False, 0.22),
+        ("  [1s — still silent]", 400, False, 0.25),
+        ("  [1.5s — verbal backchannel]", 500, False, 0.28),
+        # Learner continues
+        ("Learner: '...mercado... euh...'", 800, True, 0.1),
+        # Another pause — code-switching hesitation
+        ("  [silence — 'comment dit-on...']", 500, False, 0.20),
+        ("  [1s silence]", 500, False, 0.30),
+        ("  [1.5s — verbal backchannel]", 500, False, 0.35),
+        ("  [2s — model getting uncertain]", 500, False, 0.42),
+        # Learner continues again
+        ("Learner: '...para comprar... uma tesoura.'", 1500, True, 0.1),
+        # Final silence — this time it's a real end of turn
+        ("  [silence after complete sentence]", 300, False, 0.45),
+        ("  [600ms]", 300, False, 0.55),
+        ("  [model confidence rising]", 300, False, 0.65),
+        ("  [model confident — end of turn]", 200, False, 0.75),
+    ]
+
+    print("\nTimeline:")
+    print("-" * 70)
+
+    for description, duration_ms, is_speech, confidence in timeline:
+        print(f"\n{description} ({duration_ms}ms)")
+        # Simulate in 100ms chunks
+        n_chunks = max(1, duration_ms // 100)
+        for _ in range(n_chunks):
+            engine.feed_score(confidence, is_speech)
+            time.sleep(0.001)  # Tiny sleep for monotonic clock to advance
+
+    print("\n" + "=" * 70)
+    print(f"Total events: {len(events_log)}")
+    print(f"Final state: {engine.state.value}")
+
+    # Show CEFR comparison
+    print("\n" + "=" * 70)
+    print("CEFR LEVEL COMPARISON")
+    print("=" * 70)
+    print(f"{'Level':<6} {'Eager':<8} {'Final':<8} {'Visual BC':<10} {'Verbal BC':<10} {'Encourage':<10}")
+    print("-" * 52)
+    for level, preset in CEFR_PRESETS.items():
+        print(f"{level:<6} {preset.eager_threshold:<8.2f} {preset.final_threshold:<8.2f} "
+              f"{preset.visual_backchannel_ms:<10.0f} {preset.verbal_backchannel_ms:<10.0f} "
+              f"{preset.encouragement_ms:<10.0f}")
+    print()
+    print("A1/A2: Very patient — high final threshold (0.75-0.80), early encouragement")
+    print("B1:    Default — balanced patience and responsiveness")
+    print("B2/C1: More responsive — lower final threshold (0.60-0.65)")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.DEBUG,
+        format="%(asctime)s %(levelname)s [%(name)s] %(message)s",
+    )
+    demo_simulation()

03-finetune-pipecat-pt/README.md

@@ -0,0 +1,489 @@
+# Experimento 03 — Fine-tune Pipecat Smart Turn para Portugues + Frances
+
+> Fine-tuning do Pipecat Smart Turn v3 para **avatar conversacional de aprendizado de portugues** por francofonos. Primeiro modelo de turn-taking otimizado para aprendizes L2.
+
+---
+
+## Sumario
+
+1. [Objetivo](#objetivo)
+2. [Por que a partir do Pipecat](#por-que-a-partir-do-pipecat)
+3. [Metodo — Pipeline de dados](#metodo--pipeline-de-dados)
+   - [Pipeline de criacao de dados](#pipeline-de-criacao-de-dados)
+   - [Projetos que validam este metodo](#projetos-que-validam-este-metodo)
+4. [Fine-tuning](#fine-tuning)
+   - [Modelo base](#modelo-base)
+   - [Estrategia de fine-tuning](#estrategia-de-fine-tuning)
+   - [Infra](#infra)
+5. [Melhorias para aprendizado de idiomas (L2)](#melhorias-para-aprendizado-de-idiomas-l2)
+   - [Custo assimetrico](#custo-assimetrico)
+   - [Threshold duplo (Deepgram Flux)](#threshold-duplo-deepgram-flux)
+   - [Dados L2 reais (Speak & Improve)](#dados-l2-reais-speak--improve)
+   - [CEFR-aware presets](#cefr-aware-presets)
+6. [Sistema de backchannel](#sistema-de-backchannel)
+   - [Problema: "o avatar travou?"](#problema-o-avatar-travou)
+   - [Solucao: sinais de escuta ativa](#solucao-sinais-de-escuta-ativa)
+   - [Presets por nivel CEFR](#presets-por-nivel-cefr)
+7. [Engine de inferencia (06_inference.py)](#engine-de-inferencia-06_inferencepy)
+   - [Estados do turn-taking](#estados-do-turn-taking)
+   - [Exemplo: aprendiz B1 conjugando verbo](#exemplo-aprendiz-b1-conjugando-verbo)
+8. [Dados especificos — Frances falando portugues](#dados-especificos--frances-falando-portugues)
+   - [Tipos de hesitacao](#tipos-de-hesitacao)
+   - [Geracoes com Claude](#geracoes-com-claude)
+9. [Benchmarks de referencia](#benchmarks-de-referencia)
+   - [Pipecat Smart Turn v3.0](#pipecat-smart-turn-v30)
+   - [LiveKit v0.4.1](#livekit-v041)
+   - [Outros modelos](#outros-modelos)
+   - [Gap sintetico → real](#gap-sintetico--real)
+10. [Metricas alvo](#metricas-alvo)
+11. [Correcoes apos analise de referencias](#correcoes-apos-analise-de-referencias)
+12. [Estrutura de arquivos](#estrutura-de-arquivos)
+13. [Cronograma](#cronograma)
+14. [Dependencias](#dependencias)
+15. [Referencias](#referencias)
+
+---
+
+## Objetivo
+
+Fine-tunar o modelo pre-treinado do **Pipecat Smart Turn v3** (ja treinado em 23 linguas, 270K amostras) especificamente para:
+
+1. **Portugues brasileiro** — melhorar deteccao de fim de turno em conversas em PT-BR
+2. **Frances falando portugues** — detectar fim de turno de falantes nativos de frances que estao falando portugues (com sotaque, hesitacoes e code-switching tipicos)
+3. **Avatar conversacional L2** — o modelo sera usado em um avatar de IA que ensina portugues a francofonos, exigindo paciencia extra com pausas de aprendiz
+
+> **Pioneirismo**: Nenhum modelo de turn-taking otimizado para aprendizes L2 existe (comercial ou academico). O BabelCast e o primeiro.
+
+## Por que a partir do Pipecat
+
+Nos experimentos anteriores (`previous-experiments/02-finetune-scratch/`) treinamos do zero com Whisper Tiny + dados CORAA/MUPE. Resultados:
+
+| Metrica | Do zero (melhor) | Pipecat original (PT) |
+|---------|------------------|----------------------|
+| Accuracy | 78.2% | **95.42%** |
+| False Positive | 16.8% @0.5 | **2.79%** |
+| False Negative | 5.1% @0.5 | **1.79%** |
+| F1 | 0.798 | ~0.96 (estimado) |
+| Dados | 15K amostras | 270K amostras |
+| Labels | Pontuacao + corte artificial | LLM-curados + TTS |
+
+A diferenca e brutal: o Pipecat chega a **95.42% em portugues** com dados LLM-curados + TTS; nos chegamos a 78.2% com labels de pontuacao. O problema nao e o modelo — e a **qualidade dos dados**.
+
+**Problemas do treino do zero:**
+- Labels de baixa qualidade (pontuacao nao indica fim de turno de verdade)
+- Corte artificial em 30-75% nao simula pausas reais de hesitacao
+- Poucos dados (15K vs 270K do Pipecat)
+- Modelo nao entende pausas de hesitacao — acerta so 67.7% das pausas @threshold=0.5
+
+**Vantagens de partir do Pipecat:**
+- Modelo ja entende turn-taking em 23 linguas
+- Precisa de muito menos dados pra adaptar (5-10K vs 270K)
+- Transfer learning: mantem conhecimento geral, adapta pra PT-BR
+- Treino muito mais rapido (~30 min vs horas)
+
+## Metodo — Pipeline de dados
+
+### Pipeline de criacao de dados
+
+Baseado no pipeline comprovado do Pipecat v3.1:
+
+```
+Etapa 1: Frases fonte
+  - Transcricoes do CORAA (portugues conversacional real)
+  - Frases geradas pelo Claude (contextos especificos)
+  - Frases tipicas de aprendiz de frances falando PT
+  - Speak & Improve Corpus 2025 (340h L2 com disfluencias)
+         |
+Etapa 2: LLM processa (Claude Haiku — custo baixo)
+  - Filtra frases com erros / ambiguas (Pipecat: Gemini removeu 50-80%)
+  - Classifica: COMPLETO vs INCOMPLETO (semantico, nao por pontuacao)
+  - Insere fillers brasileiros: "hum", "tipo", "ne", "entao", "e..."
+  - Insere fillers de frances falando PT: "euh", "comment dit-on", "como se diz"
+  - Gera variantes incompletas: corta frases em pontos naturais
+         |
+Etapa 3: TTS gera audio
+  - Vozes PT-BR nativas (Kokoro, Google Chirp3)
+  - Vozes com sotaque frances (XTTS voice cloning, ou Chirp3 com accent)
+  - Variacao de velocidade, tom, ruido de fundo
+         |
+Etapa 4: Dataset final
+  - 5-10K amostras balanceadas (50% completo / 50% incompleto)
+  - Metadados: lingua, sotaque, tipo_filler, confianca_label
+```
+
+### Projetos que validam este metodo
+
+| Projeto | O que fizeram | Resultado |
+|---------|--------------|-----------|
+| **Pipecat v3.1** | Gemini filtra frases + insere fillers, Claude/GPT geram listas de fillers, Chirp3 TTS | 270K amostras, 81-97% accuracy, 23 linguas |
+| **LiveKit** | Qwen 7B como professor gera soft labels, destila pra 0.5B | 99.3% TP rate, -39% interrupcoes |
+| **Vogent Turn 80M** | Dados humanos + sinteticos com edge cases (disfluencias, listas, pausas) | 94.1% accuracy, estado da arte |
+| **Deepgram** | 100+ horas anotadas por humanos, refinamento iterativo de labels | Melhor calibracao entre todos |
+| **SpeculativeETD** | MultiWOZ texto → TTS + fillers injetados + pausas sinteticas | 120K amostras, dataset publico |
+| **SODA (Allen AI)** | GPT-3.5 gera 1.5M dialogos a partir de knowledge graph | Preferido sobre BlenderBot, Vicuna |
+| **Refuel Autolabel** | GPT-4 como anotador: 88.4% agreement (vs 86% humano) | 20x mais rapido, 7x mais barato |
+
+**Referencia principal:** [Pipecat Data Generation Contribution Guide](https://github.com/pipecat-ai/smart-turn/blob/main/docs/data_generation_contribution_guide.md)
+
+## Fine-tuning
+
+### Modelo base
+
+```python
+# Baixar modelo pre-treinado do HuggingFace
+from huggingface_hub import hf_hub_download
+model_path = hf_hub_download("pipecat-ai/smart-turn-v3", "model.onnx")
+
+# Whisper Tiny encoder (39M) + linear classifier
+# Hidden size: 384, ONNX INT8: 8MB, FP32: 32MB
+```
+
+### Estrategia de fine-tuning
+
+```python
+# Carregar pesos pre-treinados do Pipecat
+model = SmartTurnModel(whisper_model="openai/whisper-tiny")
+model.load_state_dict(pipecat_weights)
+
+# LR uniforme 5e-5 (igual ao Pipecat train.py)
+optimizer = AdamW(model.parameters(), lr=5e-5, weight_decay=0.01)
+
+# Focal Loss com custo assimetrico para aprendizes L2
+criterion = FocalLoss(gamma=2.0, alpha=0.25, fp_penalty=2.0)
+
+# 6 epochs, batch_size=128, cosine schedule + warmup 0.2
+train(model, portuguese_dataset, epochs=6, batch_size=128)
+```
+
+### Infra
+
+- **GPU**: Modal A10G (~$0.50/run)
+- **Tempo estimado**: 15-30 min
+- **Alternativa**: ai-gateway → TensorDock/Vast.ai
+
+## Melhorias para aprendizado de idiomas (L2)
+
+> Pesquisa realizada em 2026-03-16 confirmou que **nenhum modelo de turn-taking para aprendizes L2 existe** — nem comercial (Praktika, ELSA, Gliglish, TalkPal) nem academico. Todos usam abordagens genericas.
+
+### Custo assimetrico
+
+Para aprendizado de idiomas, **interromper o aluno (FP) e muito pior que esperar demais (FN)**. Um aluno interrompido perde confianca e para de tentar.
+
+```python
+# fp_penalty=2.0: falsos positivos custam 2x mais na loss
+criterion = FocalLoss(gamma=2.0, alpha=0.25, fp_penalty=2.0)
+```
+
+Inspirado no ConversAR (Meta, 2025) que usa "infinite thinking period" para L2.
+
+### Threshold duplo (Deepgram Flux)
+
+Dois thresholds separados para latencia vs precisao:
+
+| Threshold | Valor | Funcao |
+|-----------|-------|--------|
+| **eager** (0.3-0.5) | Prepara resposta do LLM especulativamente | Reduz latencia percebida |
+| **final** (0.7+) | Autoriza o avatar a falar | Minimiza interrupcoes |
+
+Se o score cai antes de atingir `final`, a preparacao especulativa e descartada — sem custo para o aluno.
+
+### Dados L2 reais (Speak & Improve)
+
+O [Speak & Improve Corpus 2025](https://huggingface.co/datasets/speak-improve/corpus) contem **340 horas** de fala L2 em ingles com anotacoes de disfluencia, niveis CEFR A2-C1.
+
+Embora seja ingles (nao portugues), os **padroes de hesitacao L2 sao cross-linguisticos**: pausas longas, repeticoes, auto-correcoes. Usamos como fonte de hesitacao patterns no fine-tuning.
+
+### CEFR-aware presets
+
+O modelo adapta paciencia conforme o nivel do aluno:
+
+| Nivel | final_threshold | eager_threshold | Comportamento |
+|-------|----------------|-----------------|---------------|
+| **A1** (iniciante) | 0.80 | 0.40 | Muito paciente — espera pausas longas |
+| **A2** | 0.75 | 0.38 | Paciente |
+| **B1** (intermediario) | 0.70 | 0.35 | Moderado |
+| **B2** | 0.65 | 0.33 | Responsivo |
+| **C1** (avancado) | 0.60 | 0.30 | Quase nativo — resposta rapida |
+
+## Sistema de backchannel
+
+### Problema: "o avatar travou?"
+
+Aprendizes L2 frequentemente pausam 1-3 segundos para pensar em conjugacoes, vocabulario, ou estrutura. Sem feedback, o aprendiz acha que o avatar travou e desiste de falar.
+
+> **Pesquisa**: Tavus identifica um threshold de **600ms** — apos esse tempo, o falante espera alguma resposta do sistema. Para L2, esse tempo e ainda mais critico.
+
+### Solucao: sinais de escuta ativa
+
+O avatar emite sinais progressivos de que esta ouvindo:
+
+| Tempo de silencio | Sinal | Tipo | Exemplo |
+|-------------------|-------|------|---------|
+| **600ms** | Aceno visual | Visual | Avatar faz "mhm" com a cabeca |
+| **1.5s** | Backchannel verbal | Audio | "mhm", "sim", "uhum" |
+| **3.0s** | Encorajamento | Audio | "sem pressa", "pode continuar", "estou ouvindo" |
+
+Os sinais **nao interrompem o turno do aluno** — sao sobreposicoes curtas que indicam escuta ativa.
+
+### Presets por nivel CEFR
+
+| Nivel | Visual (ms) | Verbal (ms) | Encorajamento (ms) |
+|-------|-------------|-------------|---------------------|
+| **A1** | 500 | 1200 | 2500 |
+| **B1** | 600 | 1500 | 3000 |
+| **C1** | 800 | 2000 | 4000 |
+
+Alunos A1 recebem sinais mais cedo; alunos C1 precisam de menos apoio.
+
+## Engine de inferencia (06_inference.py)
+
+O arquivo `06_inference.py` implementa a engine completa de turn-taking com backchannel.
+
+### Estados do turn-taking
+
+```
+LISTENING → SILENCE → BACKCHANNEL_VISUAL → BACKCHANNEL_VERBAL → ENCOURAGEMENT
+    ↑          ↓              ↓                     ↓                  ↓
+    ←──────────←──────────────←─────────────────────←──────────────────←
+    (aluno volta a falar)
+
+SILENCE → PREPARING → RESPONDING
+           (eager)      (final)
+```
+
+- **LISTENING**: aluno esta falando, modelo monitora score
+- **SILENCE**: pausa detectada, timer inicia
+- **BACKCHANNEL_***: sinais de escuta ativa (nao interrompe turno)
+- **PREPARING**: score atingiu eager_threshold, LLM comeca a preparar resposta
+- **RESPONDING**: score atingiu final_threshold, avatar fala
+
+### Exemplo: aprendiz B1 conjugando verbo
+
+```
+Aluno: "Ontem eu... [pausa 600ms]"
+Avatar: [aceno visual - nod]
+Aluno: "... fui? fiz? [pausa 1.5s]"
+Avatar: "mhm" [backchannel verbal]
+Aluno: "... fui ao mercado."
+Avatar: [espera final_threshold] → responde normalmente
+```
+
+## Dados especificos — Frances falando portugues
+
+### Tipos de hesitacao
+
+| Tipo | Exemplo | Label |
+|------|---------|-------|
+| Filler frances | "Eu fui... euh... ao mercado" | INCOMPLETO |
+| Busca de palavra | "Eu preciso de... comment dit-on... uma tesoura" | INCOMPLETO |
+| Code-switching | "Eu gosto de... enfin... tipo... de praia" | INCOMPLETO |
+| Pausa de conjugacao | "Eu... fui? fiz?... ontem" | INCOMPLETO |
+| Entonacao francesa | Frase completa mas com pitch plano (sem queda final) | COMPLETO (dificil) |
+| Ritmo silabico | Frances e syllable-timed, PT e stress-timed | Ambos |
+
+### Geracoes com Claude
+
+```
+Prompt para Claude gerar frases de aprendiz:
+
+"Gere 100 frases que um frances de nivel B1 falando portugues diria
+em uma reuniao de trabalho. Inclua:
+- Hesitacoes tipicas (euh, alors, comment dire)
+- Erros comuns de conjugacao
+- Pausas naturais pra pensar na palavra
+- Code-switching involuntario (palavras em frances no meio)
+- Frases completas com entonacao plana (sem queda de pitch)
+
+Para cada frase, indique: COMPLETO ou INCOMPLETO"
+```
+
+## Benchmarks de referencia
+
+### Pipecat Smart Turn v3.0
+
+Audio-only, 8M params:
+
+| Lingua | Accuracy | FP | FN |
+|--------|----------|-----|-----|
+| Turco | 97.10% | 1.66% | 1.24% |
+| Coreano | 96.85% | 1.12% | 2.02% |
+| Japones | 96.76% | 2.04% | 1.20% |
+| Frances | 96.01% | 1.60% | 2.39% |
+| **Portugues** | **95.42%** | **2.79%** | **1.79%** |
+| Ingles | 94.31% | 2.64% | 3.06% |
+| Espanhol | 91.97% | 4.48% | 3.55% |
+
+Fonte: [Smart Turn v3 blog](https://www.daily.co/blog/announcing-smart-turn-v3-with-cpu-inference-in-just-12ms/)
+
+### LiveKit v0.4.1
+
+Texto-only, 500M params, @99.3% TPR:
+
+| Lingua | TNR | Melhoria vs anterior |
+|--------|------|---------------------|
+| Hindi | 96.3% | +31.48% |
+| Coreano | 94.5% | +30.38% |
+| Frances | 88.9% | +33.93% |
+| **Portugues** | **87.4%** | **+45.97%** |
+| Ingles | 87.0% | +21.69% |
+
+Fonte: [LiveKit v0.4.1 blog](https://livekit.com/blog/improved-end-of-turn-model-cuts-voice-ai-interruptions-39/)
+
+### Outros modelos
+
+| Modelo | Tipo | Accuracy | Linguas | Nota |
+|--------|------|----------|---------|------|
+| Vogent Turn 80M | Audio+texto | 94.1% | 1 (EN) | Estado da arte multimodal |
+| Krisp v2 | Audio | 82.3% bal.acc | "Agnostico" | Proprietario |
+| Deepgram Flux | ASR+EoT | #1 VAQI | 1 (EN) | Proprietario |
+| VAP | Audio | 79.6% bal.acc | 3 (EN/ZH/JP) | Academico, auto-supervisionado |
+
+### Gap sintetico → real
+
+O SpeculativeETD mostrou que modelos treinados so em dados sinteticos (TTS) perdem **muito** em dados reais:
+
+| Dataset | Wav2vec F1 |
+|---------|-----------|
+| Sintetico | 94.7% |
+| Real | **30.3%** |
+
+Isso mostra que alem de gerar dados com TTS, precisamos incluir **audio real** no fine-tuning.
+
+## Metricas alvo
+
+| Metrica | Exp 02 (do zero) | Pipecat PT (ref) | Alvo (este exp) |
+|---------|------------------|------------------|-----------------|
+| Accuracy | 78.2% | 95.42% | > 96% |
+| False Positive | 16.8% | 2.79% | < 2.5% |
+| False Negative | 5.1% | 1.79% | < 2.0% |
+| F1 | 0.798 | ~0.96 | > 0.96 |
+| Tamanho modelo | 30.5 MB | 8 MB (ONNX INT8) | ~8 MB |
+| Inferencia CPU | ~12ms | 12ms | ~12ms |
+
+## Correcoes apos analise de referencias
+
+*(2026-03-16)*
+
+Baixamos 6 papers, 10 blog posts e 7 guias tecnicos (ver `references/`). A analise cruzada revelou problemas no plano original:
+
+| Mudanca | Antes | Depois | Fonte |
+|---------|-------|--------|-------|
+| Focal Loss alpha | 0.6 | **0.25** | Lin et al. 2017, Table 1a |
+| Batch size | 32 | **128** | Pipecat train.py usa 384 |
+| Epochs | 10 | **6** | Pipecat usa 4 em 270K |
+| Label smoothing | 0.05 | **removido** | EMNLP 2022: dupla regularizacao com FL |
+| Learning rate | diferencial (0.1x encoder) | **uniforme 5e-5** | Pipecat train.py usa lr unica |
+| Ruido augmentation | Gaussiano | **ruido real** (cafe/escritorio) | Pipecat v3.2: -40% erros |
+| ONNX opset | 17 | **18** | Pipecat train.py |
+| Quantizacao | nenhuma | **INT8 estatica** (entropy, 1024 calib) | Pipecat deploy: 32MB → 8MB |
+| Loss alternativa | so Focal | **Focal + BCE** comparados | Pipecat original usa BCE |
+
+### Tecnicas adicionais identificadas
+
+1. **Knowledge distillation** (LiveKit: -39% interrupcoes, +45.97% melhoria PT)
+2. **Short utterance dataset** (Pipecat v3.2: -40% erros em respostas curtas) — implementado
+3. **Audio real misturado com TTS** (SpeculativeETD: F1 cai de 94.7% → 30.3% so com sintetico) — implementado via CORAA
+4. **Pausa de 1.5-3s apos fillers** (SpeculativeETD V3: melhor variante) — implementado
+5. **Threshold per-language** (LiveKit: languages.json com thresholds por lingua)
+
+## Estrutura de arquivos
+
+```
+03-finetune-pipecat-pt/
+  README.md                    # Este documento
+  01_download_pipecat.py       # Baixa modelo pre-treinado do HuggingFace
+  02_generate_labels.py        # Claude API gera/filtra/classifica frases PT
+  03_generate_audio.py         # TTS gera audio (nativo + sotaque frances)
+  04_finetune.py               # Fine-tune com custo assimetrico + dados L2
+  05_evaluate.py               # Avaliacao com dual threshold sweep
+  06_inference.py              # Engine de inferencia com backchannel + CEFR presets
+  modal_run.py                 # Deploy no Modal (GPU)
+
+  references/
+    hesitation_turn_taking_l2_review.md    # Mini-artigo com 35 referencias
+    papers/                    # 6 PDFs academicos + summaries
+      focal_loss_lin_2017.*
+      speculative_etd_2025.*
+      vap_turn_taking_ekstedt_2024.*
+      turn_taking_review_skantze_2021.*
+      soda_dialog_distillation_2023.*
+      finite_state_turn_taking_raux_2009.*
+    papers/hesitation-l2-french/   # 19 PDFs + 10 MD (L2 hesitacao/francofono)
+    papers/language-learning-turn-taking/  # Pesquisa L2 turn-taking (2026-03-16)
+      survey_turn_taking_iwsds2025.*       # Survey IWSDS 2025
+      multilingual_vap_2024.*              # VAP multilingual
+      speak_improve_corpus_2025.*          # Speak & Improve L2 corpus
+      hesitation_tagging_l2_whisper.*      # Whisper+LoRA hesitation
+      conversar_mixed_reality_l2.*         # ConversAR (Meta Quest)
+      deepgram_flux.*                      # Deepgram Flux overview
+      hume_evi.*                           # Hume EVI overview
+      praktika_openai.*                    # Praktika case study
+      tavus_turn_taking_guide.*            # Tavus guide
+    blogs/                     # 10 blog posts
+    guides/                    # 7 technical guides
+
+  data/                        # (gitignored)
+    pipecat_pt_audio/          # Audio PT do Pipecat v3.2
+    pipecat_pt_test/           # Audio PT test do Pipecat v3.2
+    claude_labeled/            # Frases processadas pelo Claude (JSON)
+    tts_dataset/               # Audio gerado (nativo + sotaque frances)
+    noise_samples/             # Ruido real CC-0 (cafe, escritorio)
+
+  results/                     # (gitignored)
+    best_model.pt              # Modelo treinado
+    smart_turn_pt_v3.onnx      # ONNX INT8 (~8 MB)
+    smart_turn_pt_v3_fp32.onnx # ONNX FP32 (~32 MB)
+    training_results.json      # Metricas
+    evaluation_results.json    # Comparacao com baseline
+```
+
+## Cronograma
+
+| Etapa | Descricao | Tempo |
+|-------|-----------|-------|
+| 1 | Baixar e inspecionar modelo Pipecat | 1h |
+| 2 | Script de labeling com Claude API | 1 dia |
+| 3 | Gerar audio TTS (nativo PT-BR) | 1 dia |
+| 4 | Gerar audio com sotaque frances | 1-2 dias |
+| 5 | Fine-tune no Modal | 30 min |
+| 6 | Avaliacao + dual threshold sweep | 2h |
+| 7 | Teste no avatar conversacional | 1 dia |
+| **Total** | | **~5 dias** |
+
+## Dependencias
+
+```
+# Python
+torch, torchaudio, transformers    # Modelo
+anthropic                          # Claude API (labeling)
+kokoro, TTS                        # Geracao de audio
+datasets, soundfile, librosa       # Processamento
+modal                              # Deploy GPU
+
+# APIs
+ANTHROPIC_API_KEY                  # Claude Haiku pra labeling
+MODAL_TOKEN                        # Modal pra treino GPU
+
+# Modelos
+pipecat-ai/smart-turn-v3          # HuggingFace — modelo pre-treinado
+openai/whisper-tiny                # HuggingFace — encoder base
+```
+
+## Referencias
+
+Documento completo de pesquisa com 35 referencias: [`references/hesitation_turn_taking_l2_review.md`](references/hesitation_turn_taking_l2_review.md)
+
+**Principais:**
+
+| # | Referencia | Contribuicao |
+|---|-----------|--------------|
+| 1 | Pipecat Smart Turn v3 (Daily, 2025) | Modelo base, pipeline de dados |
+| 2 | Lin et al. (ICCV 2017) | Focal Loss, alpha=0.25 |
+| 3 | Skantze (CSL 2021) | Survey de turn-taking |
+| 4 | Knill et al. (2025) | Speak & Improve L2 corpus |
+| 5 | Saeki et al. (2025) | Whisper+LoRA hesitation tagging |
+| 6 | Gamboa et al. (2025) | ConversAR, custo assimetrico L2 |
+| 7 | Deepgram Flux (2025) | Dual threshold, speculative ASR |
+| 8 | Ekstedt et al. (2024) | VAP multilingual |
+| 9 | Raux & Eskenazi (NAACL 2009) | FSM turn-taking |
+| 10 | LiveKit (2025) | Knowledge distillation, 500M params |

03-finetune-pipecat-pt/modal_run.py

@@ -0,0 +1,267 @@
+"""Run the full fine-tuning pipeline on Modal (A10G GPU).
+
+Usage:
+    modal run modal_run.py
+
+This deploys the pipeline on a Modal A10G GPU (~$0.50/run):
+1. Downloads Pipecat Portuguese data
+2. Generates labels with Claude API (if ANTHROPIC_API_KEY set)
+3. Generates TTS audio with Kokoro
+4. Fine-tunes SmartTurnV3Model
+5. Evaluates against Pipecat baseline
+6. Saves results to Modal volume
+
+Estimated time: 30-60 min total
+Estimated cost: ~$0.50-1.00
+"""
+
+from __future__ import annotations
+
+import modal
+
+app = modal.App("babelcast-finetune-pipecat-pt")
+
+# Persistent volume for data + results
+volume = modal.Volume.from_name("finetune-pipecat-pt", create_if_missing=True)
+
+# GPU image with all dependencies
+image = (
+    modal.Image.debian_slim(python_version="3.11")
+    .pip_install(
+        # Core ML
+        "torch>=2.1",
+        "torchaudio>=2.1",
+        "transformers>=4.36",
+        "datasets>=2.16",
+        # Audio processing
+        "soundfile>=0.12",
+        "librosa>=0.10",
+        "numpy<2",
+        # TTS
+        "kokoro>=0.3",
+        # ONNX for evaluation
+        "onnxruntime>=1.16",
+        # Claude API for labeling
+        "anthropic>=0.40",
+        # Utilities
+        "huggingface-hub>=0.20",
+    )
+    .apt_install("ffmpeg", "libsndfile1")
+)
+
+
+@app.function(
+    image=image,
+    gpu="A10G",
+    timeout=3600,  # 1 hour max
+    volumes={"/workspace": volume},
+    secrets=[
+        modal.Secret.from_name("anthropic-api-key", required=False),
+        modal.Secret.from_name("huggingface-token", required=False),
+    ],
+)
+def run_pipeline(
+    skip_download: bool = False,
+    skip_labels: bool = False,
+    skip_audio: bool = False,
+    skip_train: bool = False,
+    max_pipecat_samples: int = 5000,
+    max_tts_samples: int = 10000,
+    max_l2_samples: int = 2000,
+    epochs: int = 6,
+    batch_size: int = 128,
+    lr: float = 5e-5,
+    loss_fn: str = "focal",
+    fp_penalty: float = 2.0,
+):
+    """Run the full pipeline on Modal GPU."""
+    import logging
+    import os
+    import sys
+    from pathlib import Path
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s [%(name)s] %(message)s",
+    )
+    log = logging.getLogger("modal_run")
+
+    # Add script directory to path
+    script_dir = Path(__file__).parent
+    sys.path.insert(0, str(script_dir))
+
+    # Symlink data dir to /workspace for persistence
+    data_dir = script_dir / "data"
+    workspace_data = Path("/workspace/data")
+    workspace_data.mkdir(parents=True, exist_ok=True)
+    if not data_dir.exists():
+        data_dir.symlink_to(workspace_data)
+
+    # Step 1: Download Pipecat data
+    if not skip_download:
+        log.info("=" * 60)
+        log.info("STEP 1: Download Pipecat Portuguese data")
+        log.info("=" * 60)
+        from import_module_01 import download_pipecat_dataset, download_pipecat_test_data, download_onnx_model
+        try:
+            # Use importlib since filenames start with numbers
+            import importlib.util
+            spec = importlib.util.spec_from_file_location("dl", script_dir / "01_download_pipecat.py")
+            dl = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(dl)
+
+            dl.download_pipecat_dataset(max_pt_samples=max_pipecat_samples)
+            dl.download_pipecat_test_data(max_pt_samples=2000)
+            dl.download_onnx_model()
+        except Exception as e:
+            log.error("Download failed: %s", e)
+            raise
+
+        volume.commit()
+        log.info("Step 1 complete — data saved to volume")
+
+    # Step 2: Generate labels (requires ANTHROPIC_API_KEY)
+    if not skip_labels:
+        log.info("=" * 60)
+        log.info("STEP 2: Generate labels with Claude API")
+        log.info("=" * 60)
+        try:
+            import importlib.util
+            spec = importlib.util.spec_from_file_location("labels", script_dir / "02_generate_labels.py")
+            labels = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(labels)
+
+            api_key = os.environ.get("ANTHROPIC_API_KEY")
+            if api_key:
+                log.info("ANTHROPIC_API_KEY found — using Claude for labeling")
+            else:
+                log.info("No ANTHROPIC_API_KEY — using rule-based fallback")
+
+            labels.run_full_pipeline(max_transcripts=5000, max_fr_sentences=500)
+        except Exception as e:
+            log.warning("Label generation failed: %s — continuing without custom labels", e)
+
+        volume.commit()
+
+    # Step 3: Generate TTS audio
+    if not skip_audio:
+        log.info("=" * 60)
+        log.info("STEP 3: Generate TTS audio")
+        log.info("=" * 60)
+        try:
+            import importlib.util
+            spec = importlib.util.spec_from_file_location("audio", script_dir / "03_generate_audio.py")
+            audio = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(audio)
+
+            audio.run_audio_generation(
+                max_native_sentences=3000,
+                max_french_sentences=1000,
+                augment_copies=2,
+            )
+        except Exception as e:
+            log.warning("TTS generation failed: %s — continuing with Pipecat data only", e)
+
+        volume.commit()
+
+    # Step 4: Fine-tune
+    if not skip_train:
+        log.info("=" * 60)
+        log.info("STEP 4: Fine-tune SmartTurnV3Model")
+        log.info("=" * 60)
+        try:
+            import importlib.util
+            spec = importlib.util.spec_from_file_location("ft", script_dir / "04_finetune.py")
+            ft = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(ft)
+
+            ft.train(
+                epochs=epochs,
+                batch_size=batch_size,
+                lr=lr,
+                warmup_ratio=0.2,
+                max_pipecat_samples=max_pipecat_samples,
+                max_tts_samples=max_tts_samples,
+                max_l2_samples=max_l2_samples,
+                loss_fn=loss_fn,
+                fp_penalty=fp_penalty,
+            )
+        except Exception as e:
+            log.error("Training failed: %s", e)
+            raise
+
+        volume.commit()
+
+    # Step 5: Evaluate
+    log.info("=" * 60)
+    log.info("STEP 5: Evaluate")
+    log.info("=" * 60)
+    try:
+        import importlib.util
+        spec = importlib.util.spec_from_file_location("ev", script_dir / "05_evaluate.py")
+        ev = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(ev)
+
+        results_dir = Path("/workspace/results")
+        model_path = results_dir / "best_model.pt"
+        if model_path.exists():
+            results = ev.run_evaluation(model_path)
+            log.info("Evaluation complete!")
+        else:
+            log.warning("No model found at %s — skipping evaluation", model_path)
+    except Exception as e:
+        log.error("Evaluation failed: %s", e)
+
+    volume.commit()
+    log.info("=" * 60)
+    log.info("PIPELINE COMPLETE — results saved to Modal volume 'finetune-pipecat-pt'")
+    log.info("=" * 60)
+
+
+@app.function(
+    image=image,
+    volumes={"/workspace": volume},
+)
+def download_results(local_dir: str = "results") -> list[str]:
+    """Download results from Modal volume."""
+    from pathlib import Path
+    import shutil
+
+    results_dir = Path("/workspace/results")
+    if not results_dir.exists():
+        print("No results found on volume")
+        return []
+
+    files = []
+    for f in results_dir.rglob("*"):
+        if f.is_file():
+            files.append(str(f.relative_to(results_dir)))
+            print(f"  {f.name}: {f.stat().st_size / 1024:.1f} KB")
+
+    return files
+
+
+@app.local_entrypoint()
+def main(
+    skip_download: bool = False,
+    skip_labels: bool = False,
+    skip_audio: bool = False,
+    skip_train: bool = False,
+    download_only: bool = False,
+    epochs: int = 10,
+    batch_size: int = 32,
+):
+    """Entry point for `modal run modal_run.py`."""
+    if download_only:
+        files = download_results.remote()
+        print(f"\nFound {len(files)} result files on volume")
+        return
+
+    run_pipeline.remote(
+        skip_download=skip_download,
+        skip_labels=skip_labels,
+        skip_audio=skip_audio,
+        skip_train=skip_train,
+        epochs=epochs,
+        batch_size=batch_size,
+    )

03-finetune-pipecat-pt/references/blogs/assemblyai_turn_detection.md

@@ -0,0 +1,137 @@
+---
+title: "How Intelligent Turn Detection (Endpointing) Solves the Biggest Challenge in Voice Agent Development"
+source: https://www.assemblyai.com/blog/turn-detection-endpointing-voice-agent
+date_accessed: 2026-03-16
+---
+
+# How Intelligent Turn Detection (Endpointing) Solves the Biggest Challenge in Voice Agent Development
+
+## Introduction
+
+Voice agents enable natural conversations between humans and AI systems, representing one of Speech AI's fastest-growing applications. However, a persistent challenge affects all voice agent implementations: **turn detection**, or determining when a human finishes speaking so the AI can respond appropriately.
+
+## The Latency Problem in Voice Agents
+
+Voice agent developers prioritize low latency for optimal user experience. Three fundamental latencies exist in streaming speech-to-text models:
+
+1. **Partial transcript latency** -- Speed of returning initial transcript predictions
+2. **Final transcript latency** -- Speed of returning finalized transcripts after speech ends
+3. **Endpointing latency** -- Speed of detecting when someone stops speaking
+
+Voice agents specifically require fast endpointing latency. Developers have historically over-optimized for general latency reduction, treating end-of-turn detection as secondary. However, addressing turn detection provides "far greater improvements to the user experience than incremental latency optimizations," as endpointing delay exists at a different magnitude than millisecond-level latency gains.
+
+## Three Endpointing Methods
+
+### Manual Endpointing
+Users explicitly indicate completion through button presses or voice commands. This creates poor user experience and harms adoption despite potentially fast response times.
+
+### Silence Detection
+The current industry standard waits for specified silence duration thresholds. This approach balances responsiveness against premature interruptions but struggles finding optimal threshold values -- long enough to avoid cutoffs mid-thought, yet short enough to maintain conversational fluidity.
+
+### Semantic Endpointing
+Advanced systems analyze what someone says rather than just silence duration. This approach understands when thoughts are complete, distinguishing natural pauses mid-sentence from genuine utterance endings. Modern implementations use language models to predict sentence boundaries and semantic completeness.
+
+## How Semantic Endpointing Works
+
+Semantic endpointing encompasses multiple implementation approaches. AssemblyAI's Universal-Streaming model predicts a special token that the model learns to identify during training based on context.
+
+### Configuration Parameters
+
+Several user-configurable options enable developers to tune endpointing behavior:
+
+- **end_of_turn_confidence_threshold** (default: 0.7) -- Required confidence level for end-of-turn predictions
+- **min_end_of_turn_silence_when_confident** (default: 160ms) -- Minimum silence duration after confident prediction to prevent false positives
+- **max_turn_silence** (default: 2400ms) -- Fallback silence-based endpointing for unusual speech patterns
+
+The system requires three conditions for semantic end-of-turn:
+
+1. Model predicts end-of-turn with sufficient confidence
+2. Minimum silence duration has passed
+3. Minimal speech present to prevent false positives from audio noise
+
+Traditional silence-based endpointing remains enabled as a final catch-all mechanism for atypical patterns.
+
+## Comparative Analysis: Turn Detection Approaches
+
+### LiveKit: Semantic-Only Approach
+
+**Input:** Transcribed text only
+**Key Dependency:** Voice Activity Detection (VAD)
+**Processing:** Runs after VAD detects speech end
+
+**Strengths:**
+- Simple, focused semantic analysis
+- Open source customization available
+
+**Limitations:**
+- Heavy VAD dependency creates delays if background noise triggers continuous VAD
+- Ignores audio cues humans naturally use for turn boundaries
+- Tends toward maximum delay periods, creating sluggish conversations
+
+### Pipecat: Audio-Centric Detection
+
+**Input:** Audio features only (prosody, intonation)
+**Key Dependency:** Direct audio analysis
+**Processing:** Real-time audio pattern recognition
+
+**Strengths:**
+- Leverages natural prosodic and intonation cues
+- Potentially faster turn detection than text-dependent models
+- Open source flexibility
+
+**Limitations:**
+- Sensitive to background noise interference
+- Performance varies significantly with speaker accents
+- Struggles with atypical prosodic patterns
+- Limited by audio quality availability
+
+### AssemblyAI: Hybrid Approach
+
+**Input:** Both transcribed text and audio context
+**Key Dependency:** Integrated approach with dynamic thresholds
+**Processing:** Context-aware analysis enabling early turn detection during silence based on syntactic completeness
+
+**Strengths:**
+- Robust across varying acoustic conditions
+- Dynamic adaptation based on sentence completeness rather than static VAD
+- Better background noise handling through semantic backup
+- More natural conversation flow with context-aware timing
+
+**Limitations:**
+- Closed source limits customization
+- Potentially higher computational requirements
+- Depends on transcription quality for optimal performance
+
+## Feature Comparison Matrix
+
+| Feature | LiveKit | Pipecat | AssemblyAI |
+|---------|---------|---------|-----------|
+| Input Type | Text only | Audio only | Text + Audio |
+| VAD Dependency | High | None | Low |
+| Noise Robustness | Poor (via VAD) | Poor | Good |
+| Speaker Variation | Good | Poor | Good |
+| Response Speed | Slow | Fast | Adaptive |
+| Complexity | Low | Medium | High |
+
+## Selection Guidance
+
+**Choose LiveKit if:**
+- You need simple, semantic-focused approaches
+- You have clean audio environments with minimal background noise
+- Response speed is less critical than accuracy
+
+**Choose Pipecat if:**
+- You have consistent speaker profiles and clean audio
+- You want to experiment with pure audio-based detection
+
+**Choose AssemblyAI if:**
+- You want combined semantic and acoustic endpointing
+- You handle diverse speakers and noisy environments
+- Natural conversation flow is critical
+- Deployment flexibility matters
+
+## The Future of Turn Detection
+
+Single-modality solutions offer simplicity and specialized performance, while hybrid approaches like AssemblyAI's demonstrate benefits of combining multiple signal types for robust and natural conversational experiences. Future developments will likely incorporate conversational context, speaker identification, and multimodal cues (including visual information in relevant scenarios).
+
+The optimal choice depends on specific use case requirements, technical constraints, and acceptable trade-offs between accuracy, speed, and robustness.

03-finetune-pipecat-pt/references/blogs/deepgram_eot_evaluation.md

@@ -0,0 +1,58 @@
+---
+title: "Evaluating End-of-Turn Detection Models"
+source: https://deepgram.com/learn/evaluating-end-of-turn-detection-models
+date_accessed: 2026-03-16
+---
+
+# Evaluating End-of-Turn (Turn Detection) Models
+
+## Overview
+
+Natural voice agent interactions depend critically on accurate turn-taking behavior. End-of-Turn (EoT) detection -- predicting when a speaker has finished and awaits a response -- is essential for creating conversational agents that feel responsive rather than interruptive.
+
+Deepgram developed a comprehensive evaluation methodology for turn detection after finding existing approaches inadequate. Their approach prioritizes real-world performance over proxy metrics, evaluating "complete conversations between humans" rather than isolated turns.
+
+## Full Conversational Evaluation
+
+Rather than testing individual turns, Deepgram analyzes entire conversations. This methodology reflects realistic scenarios where:
+
+- **Natural labeling**: Conversation partners provide high-quality, low-latency ground truth detection
+- **Variable detection budgets**: Different turns demand faster responses based on context (simple queries versus complex reasoning)
+- **Natural pause patterns**: Pre-turn silences enable evaluation of start-of-speech detection and backchannel phenomena
+
+The team labeled over 100 hours of real conversations with ground truth transcripts and timestamped EoT labels. Notably, refining their annotation specification from "End-of-Thought" to "End-of-Turn" improved annotator confidence from 5/10 to 8.5/10 by reducing ambiguity.
+
+## The Challenge with Timestamps
+
+Human annotations revealed an unexpected problem: annotators consistently left small gaps between speech completion and label placement. Deepgram discovered their system frequently detected EoT within these gaps -- appearing as false positives under naive temporal alignment.
+
+### Forced Alignment Solution
+
+Rather than trusting raw human timestamps, Deepgram applied "forced alignment to update the human timestamps," extracting more precise end-of-speech markers. While conservative human placement might reflect natural pauses before responses, voice agents benefit from fastest possible detection since "their turn starts are frequently delayed relative to EoT detection due to extra latency associated with LLM and TTS generation."
+
+## Sequence Alignment for Improved Evaluation
+
+Standard temporal matching proved insufficiently precise. Deepgram adopted sequence alignment -- treating turn boundaries as special tokens (`[EoT]`) within transcripts, similar to Word Error Rate (WER) calculation.
+
+**Impact**: This shift yielded "3-5% absolute increases in precision and recall across models," with "manual investigation of results suggested the new values were more representative of the true performance."
+
+The approach handled all detector types: all-in-one STT solutions, text-based detectors, and audio-only models (using Nova-3 for inter-turn transcription).
+
+## Handling Dropped Turns
+
+A modified Levenshtein algorithm addressed cases where intermediate turns were missed. Beyond simple edit distance, the system "determines the best alignment not only based on overall edit distance but also based on the most likely alignment between EoT tokens," preventing misalignment when turns are dropped.
+
+## Start-of-Turn (SoT) Evaluation
+
+Complementing EoT analysis, Deepgram evaluated "start-of-turn (SoT)" detection -- identifying when users resume speaking, critical for handling interruptions or "barge-in" scenarios.
+
+**Flux performance metrics**:
+- Detection within ~100-200ms of first word onset
+- False positive rate: less than or equal to 1-2%
+- Analysis used word start times as detection targets (representing unachievable lower bounds)
+
+Negative latency directly indicates false positives -- detecting speech before it actually begins.
+
+## Future Directions
+
+The team highlighted emerging evaluation frontiers: "one could combine the various aspects of turn detection, such as false positive rate and latency, into single quality metrics." They specifically mentioned their Voice Agent Quality Index (VAQI) framework and anticipated incorporating semantic/conversational flow considerations -- enabling "stratified or weighted metrics that reflect that some turns merit faster responses than others."

03-finetune-pipecat-pt/references/blogs/krisp_turn_taking_v1.md

@@ -0,0 +1,78 @@
+---
+title: "Turn-Taking for Voice AI"
+source: https://krisp.ai/blog/turn-taking-for-voice-ai/
+date_accessed: 2026-03-16
+---
+
+# Audio-only, 6M Weights Turn-Taking Model for Voice AI Agents
+
+## Overview
+
+Krisp has developed a lightweight turn-taking model designed to determine when speakers should transition in voice conversations. This technology is included at no additional cost in Krisp's VIVA SDK.
+
+## What is Turn-Taking?
+
+Turn-taking represents "the fundamental mechanism by which participants in a conversation coordinate who speaks when." In voice AI contexts, it manages when agents should listen, speak, or remain silent. The model addresses two primary tasks:
+
+- **End-of-turn prediction**: Detecting when the current speaker will finish
+- **Backchannel prediction**: Recognizing brief acknowledgments like "uh-huh" without speaker transitions
+
+## Implementation Approaches
+
+**Audio-based methods** analyze acoustic features including pitch variations, energy levels, intonation, pauses, and speaking rate. These enable low-latency responses critical for real-time scenarios.
+
+**Text-based approaches** examine transcribed content, identifying linguistic cues like sentence boundaries and discourse markers, though they typically require larger architectures.
+
+**Multimodal (fusion) systems** combine both modalities, leveraging acoustic cues alongside semantic understanding.
+
+## Key Challenges
+
+- **Hesitation vs. completion**: Distinguishing filler words ("um," "you know") from true turn endings
+- **Natural pauses**: Differentiating conversational pauses from actual turn boundaries
+- **Response speed**: Minimizing latency while maintaining accuracy
+- **Speaking diversity**: Accounting for varying rhythms, accents, and intonation patterns
+
+## Model Architecture
+
+The Krisp model processes 100ms audio frames, outputting confidence scores (0-1) indicating shift likelihood. A configurable threshold determines binary shift predictions, with a default 5-second maximum hold duration.
+
+## Performance Comparison
+
+| Attribute | Krisp TT | SmartTurn v1 | SmartTurn v2 | VAD-based |
+|-----------|----------|-------------|-------------|-----------|
+| Parameters | 6.1M | 581M | 95M | 260k |
+| Model Size | 65 MB | 2.3 GB | 360 MB | 2.3 MB |
+| Execution | CPU | GPU | GPU | CPU |
+
+## Evaluation Metrics
+
+### Accuracy Results
+
+| Model | Balanced Accuracy | AUC Shift | F1 Score Shift | F1 Score Hold | AUC (MST vs FPR) |
+|-------|------------------|-----------|----------------|---------------|-----------------|
+| Krisp TT | 0.82 | 0.89 | 0.80 | 0.83 | 0.21 |
+| VAD-based | 0.59 | -- | 0.48 | 0.70 | -- |
+| SmartTurn V1 | 0.78 | 0.86 | 0.73 | 0.84 | 0.39 |
+| SmartTurn V2 | 0.78 | 0.83 | 0.76 | 0.78 | 0.44 |
+
+### Training Data
+
+- Approximately 2,000 hours of conversational speech
+- Around 700,000 speaker turns
+- Test dataset: 1,875 manually labeled audio samples
+
+## Key Performance Findings
+
+The Krisp model achieves "considerably faster average response time (0.9 vs. 1.3 seconds at a 0.06 FPR) compared to SmartTurn" while being 5-10 times smaller and optimized for CPU execution.
+
+## Future Development
+
+**Planned enhancements include:**
+
+- Text-based turn-taking using custom neural networks
+- Multimodal audio-text fusion for improved accuracy
+- Backchannel detection to distinguish meaningful interruptions from casual listening acknowledgments
+
+---
+
+*Article published August 5, 2025 by Krisp Engineering Team*

03-finetune-pipecat-pt/references/blogs/krisp_turn_taking_v2.md

@@ -0,0 +1,54 @@
+---
+title: "Krisp Turn-Taking v2 - Voice AI VIVA SDK"
+source: https://krisp.ai/blog/krisp-turn-taking-v2-voice-ai-viva-sdk/
+date_accessed: 2026-03-16
+---
+
+# Audio-Only Turn-Taking Model v2 - Krisp
+
+## Overview
+
+Krisp has released Turn-Taking v2, an updated model for detecting end-of-turns in real-time conversational AI systems. The model processes audio input only, making it suitable for "human-bot interactions" and other voice AI applications integrated through Krisp's VIVA SDK.
+
+## Technical Architecture
+
+The latest iteration, **krisp-viva-tt-v2**, represents a substantial advancement over its predecessor. According to the engineering team, it was "trained on a more diverse and better-structured dataset, with richer data augmentations that help the model perform more reliably in real-world conditions."
+
+## Key Improvements
+
+- Enhanced robustness in noisy environments
+- Superior accuracy when combined with Krisp's Voice Isolation models
+- Faster and more stable turn detection during live conversations
+
+## Performance Benchmarks
+
+### Clean Audio Testing
+
+Testing evaluated approximately 1,800 real conversation samples (~1,000 "hold" cases and ~800 "shift" cases) with mild background noise:
+
+| Model | Balanced Accuracy | AUC | F1 Score |
+|-------|-------------------|-----|----------|
+| krisp-viva-tt-v1 | 0.82 | 0.89 | 0.804 |
+| **krisp-viva-tt-v2** | **0.823** | **0.904** | **0.813** |
+
+### Noisy Audio Testing (5-15 dB noise levels)
+
+| Model | Balanced Accuracy | AUC | F1 Score |
+|-------|-------------------|-----|----------|
+| krisp-viva-tt-v1 | 0.723 | 0.799 | 0.71 |
+| **krisp-viva-tt-v2** | **0.768** | **0.842** | **0.757** |
+
+V2 demonstrated "up to a 6% improvement in F1 score under noisy conditions."
+
+### Post-Processing with Voice Isolation
+
+After applying background noise and voice removal through krisp-viva-tel-v2:
+
+| Model | Balanced Accuracy | AUC | F1 Score |
+|-------|-------------------|-----|----------|
+| krisp-viva-tt-v1 | 0.787 | 0.854 | 0.775 |
+| **krisp-viva-tt-v2** | **0.816** | **0.885** | **0.808** |
+
+## Availability
+
+The model is now available as part of Krisp's VIVA SDK, designed for developers building Voice AI agents and conversational systems.

03-finetune-pipecat-pt/references/blogs/livekit_eot_v0_4.md

@@ -0,0 +1,104 @@
+---
+title: "Improved End-of-Turn Model Cuts Voice AI Interruptions 39%"
+source: https://livekit.com/blog/improved-end-of-turn-model-cuts-voice-ai-interruptions-39/
+date_accessed: 2026-03-16
+---
+
+# Improved End-of-Turn Model Cuts Voice AI Interruptions 39%
+
+## Overview
+
+LiveKit released an updated transformer-based end-of-turn detection model, `v0.4.1-intl`, featuring "a 39.23% relative reduction in false-positive interruptions" compared to the previous version. The model now emphasizes structured data handling and multilingual performance across supported languages.
+
+## Key Improvements
+
+### Performance Metrics
+
+The benchmark demonstrates consistent gains across all tested languages:
+
+| Language | v0.3.0 Error Rate | v0.4.1 Error Rate | Relative Improvement |
+|----------|------------------|------------------|----------------------|
+| Chinese | 18.70% | 13.40% | 28.34% |
+| Dutch | 26.10% | 11.90% | 54.33% |
+| English | 16.60% | 13.00% | 21.69% |
+| French | 16.80% | 11.10% | 33.93% |
+| German | 23.40% | 12.20% | 47.86% |
+| Hindi | 5.40% | 3.70% | 31.48% |
+| Indonesian | 17.00% | 10.60% | 37.65% |
+| Italian | 20.10% | 14.90% | 25.87% |
+| Japanese | 19.70% | 11.20% | 43.15% |
+| Korean | 7.90% | 5.50% | 30.38% |
+| Portuguese | 23.30% | 12.60% | 45.97% |
+| Russian | 19.50% | 12.00% | 38.46% |
+| Spanish | 21.50% | 14.00% | 33.88% |
+| Turkish | 25.40% | 12.70% | 50.0% |
+| **All** | **18.66%** | **11.34%** | **39.23%** |
+
+Error rate represents the false positive rate at a fixed true positive rate of 99.3%.
+
+## Technical Architecture
+
+### The Challenge of End-of-Turn Detection
+
+Voice AI systems must detect speech completion using three primary cues:
+
+- **Semantic content**: Word meaning and language understanding
+- **Context**: Dialogue history and conversational flow
+- **Prosody**: Tone, pauses, and rhythmic patterns
+
+The model uses an LLM backbone to effectively combine content and context information.
+
+### Structured Data Handling
+
+A primary enhancement addresses structured information collection. When users provide phone numbers, emails, or addresses, natural speech markers (intonation changes, grammatical endings) are absent. The updated model "addresses this by inferring expected formats from the agent's prompt," enabling:
+
+- Detection of complete phone numbers (typically 10 digits for US numbers)
+- Email pattern recognition
+- Address validation including street, city, state, and zip code
+
+The visualization tool demonstrates the improvement: `v0.3.0-intl` incorrectly detected end points after individual digits, while `v0.4.1-intl` waited for the complete sequence.
+
+### Handling Speech-to-Text Variability
+
+Training data incorporated multiple STT output formats. One engine might transcribe "forty two" as words while another outputs "42" numerically. By training across "common STT output formats," the model maintains consistent performance across different provider implementations.
+
+### Multilingual Generalization
+
+Despite structured data enhancements targeting English training data, improvements transferred across languages -- particularly for phone number and address detection in Spanish, French, and other languages. This stems from the Qwen2.5 base model's multilingual pretraining, which encodes "knowledge of global formats."
+
+## Model Architecture & Optimization
+
+### Base Model Selection
+
+The team selected **Qwen2.5-0.5B-Instruct** for its balance of performance and low-latency CPU inference capabilities.
+
+### Knowledge Distillation
+
+To improve multilingual accuracy without sacrificing efficiency:
+
+1. A larger **Qwen2.5-7B-Instruct** teacher model was trained
+2. Knowledge was distilled into the smaller 0.5B student model
+3. The distilled model achieves "higher multilingual accuracy with the efficiency of the smaller size"
+
+Training curves show the distilled model outperforming the baseline 0.5B version and approaching teacher performance after approximately 1,500 steps.
+
+## Availability & Deprecation
+
+- **Deployment**: Available in Agents Python 1.3.0 and Agents JS 1.0.19
+- **Model**: `MultilingualModel` now recommended for all use cases
+- **Deprecation**: The legacy `EnglishModel` is being deprecated as the multilingual version "not only matches but in most cases exceeds the performance"
+
+## Observability Integration
+
+LiveKit Agents now include built-in observability for end-of-turn detection. When Agent Observability is enabled, "every turn detection decision is logged with the exact input the model saw." Production debugging accesses `eou_detection` traces showing full prediction context.
+
+## Future Directions
+
+Current implementation relies on transcribed text. Future iterations will integrate raw audio features like pauses and emphasis through multimodal architectures, "fusing prosody directly into predictions for more precise detection."
+
+---
+
+**Resources**:
+- [GitHub Repository](https://github.com/livekit/agents)
+- [Visualization Tool](https://huggingface.co/spaces/livekit/eot-visualization)
+- [Voice AI Quickstart](https://docs.livekit.io/agents/start/voice-ai/)

03-finetune-pipecat-pt/references/blogs/livekit_transformer_turn_detection.md

@@ -0,0 +1,87 @@
+---
+title: "Using a Transformer to Improve End of Turn Detection"
+source: https://livekit.com/blog/using-a-transformer-to-improve-end-of-turn-detection
+date_accessed: 2026-03-16
+---
+
+# Using a Transformer to Improve End of Turn Detection
+
+**Published:** December 20, 2024 | **Author:** Russ D'Sa | **Reading Time:** 6 minutes
+
+## Overview
+
+End-of-turn detection represents one of the most challenging problems in voice AI applications. The core task involves determining when a user has finished speaking, allowing an AI model to respond without unintentionally interrupting the user.
+
+## Current Approach: Phrase Endpointing
+
+The predominant technique for turn detection is phrase endpointing, typically implemented through voice activity detection (VAD). This method operates as follows:
+
+- Audio packets are processed through a neural network to determine if human speech is present
+- If speech is detected, the user continues speaking
+- If silence is detected, a timer begins tracking the duration of the absence of detectable human speech
+- Once a configured silence threshold passes, an end-of-turn event triggers, allowing LLM inference to begin
+
+LiveKit Agents uses "Silero VAD to detect voice activity and provide timing parameters" to adjust sensitivity. The framework introduces a configurable delay via the `min_endpointing_delay` parameter (default: 500ms) between when VAD detects speech cessation and when LLM inference begins.
+
+## The VAD Limitation
+
+VAD only detects *when* someone is speaking based on audio signal presence. Humans, however, use semantic understanding -- analyzing what is said and how it's expressed -- to determine turn-taking. For example, "I understand your point, but..." would trigger VAD's end-of-turn signal despite the human listener recognizing the speaker intends to continue.
+
+## The EOU (End of Utterance) Model
+
+LiveKit released "an open source transformer model that uses the content of speech to predict when a user has" finished speaking. The model incorporates semantic understanding into turn detection.
+
+### Model Architecture
+
+- **Base Model:** 135M parameter transformer based on SmolLM v2 from HuggingFace
+- **Design Choice:** Small model size enables CPU-based, real-time inference
+- **Context Window:** Sliding window of the last four conversational turns
+- **Language Support:** Currently English transcriptions only; additional languages planned
+
+### How It Works
+
+During user speech, transcriptions from a speech-to-text (STT) service are appended word-by-word to the model's context window. For each final STT transcription, the model generates a confidence-level prediction regarding whether the current context represents turn completion.
+
+The model integrates with VAD by dynamically adjusting the silence timeout: longer silence periods are permitted when EOU indicates the user hasn't finished speaking, reducing interruptions while maintaining responsiveness.
+
+## Performance Results
+
+Compared to VAD alone, the EOU + VAD approach achieves:
+
+- **85% reduction** in unintentional interruptions
+- **3% false negative rate** (incorrectly indicating turn continuation)
+
+### Use Cases
+
+The model proves particularly valuable for conversational AI and customer support applications requiring data collection:
+
+- Conducting interviews
+- Collecting addresses for shipments
+- Gathering phone numbers
+- Processing payment information
+- Ordering transactions (pizza ordering demonstrated)
+
+## Implementation
+
+The turn detector is packaged as a LiveKit Agents plugin, enabling simple integration through a single additional parameter in the `VoicePipelineAgent` constructor:
+
+```
+turn_detector=TurnDetector.from_plugin()
+```
+
+Example implementation code is available in the LiveKit agents repository.
+
+## Future Development
+
+Planned improvements include:
+
+- **Multi-language Support:** Extending EOU beyond English
+- **Inference Optimization:** Reducing current ~50ms inference latency
+- **Context Window Expansion:** Increasing the four-turn window
+- **Audio-Based Detection:** Developing models for multimodal systems that process audio directly, accounting for prosodic features like intonation and cadence
+
+The team acknowledges that EOU's text-based training limits its use with natively multimodal models such as OpenAI's Realtime API. An audio-native model is under development to address this constraint.
+
+## Broader Implications
+
+The researchers emphasize that "even humans don't get this right all the time" and that non-verbal cues improve human turn-taking performance. They consider this research essential for developing natural, humanlike AI interactions and anticipate significant community innovation in this area.

03-finetune-pipecat-pt/references/blogs/pipecat_smart_turn_v3.md

@@ -0,0 +1,75 @@
+---
+title: "Announcing Smart Turn v3: CPU Inference in Just 12ms"
+source: https://www.daily.co/blog/announcing-smart-turn-v3-with-cpu-inference-in-just-12ms/
+date_accessed: 2026-03-16
+---
+
+# Smart Turn v3: CPU Inference Breakthrough
+
+## Overview
+
+Daily announced Smart Turn v3, a dramatically improved voice turn detection model featuring unprecedented efficiency. The system achieves "CPU inference in just 12ms" while maintaining open-source accessibility for weights, training data, and scripts.
+
+## Key Improvements
+
+### Size and Performance
+- **Model size:** Reduced to 8 MB (nearly 50x smaller than v2)
+- **CPU inference speed:** 12ms on modern processors; 60ms on budget AWS instances
+- **No GPU required:** Runs directly within Pipecat Cloud instances
+
+### Language Coverage
+Expanded to 23 languages including Arabic, Bengali, Chinese, Danish, Dutch, German, English, Finnish, French, Hindi, Indonesian, Italian, Japanese, Korean, Marathi, Norwegian, Polish, Portuguese, Russian, Spanish, Turkish, Ukrainian, and Vietnamese.
+
+### Architecture
+- **Foundation:** Whisper Tiny encoder (39M parameters)
+- **Classification layers:** Adapted from Smart Turn v2
+- **Total parameters:** 8M
+- **Optimization:** int8 quantization via static QAT
+- **Export format:** ONNX
+
+## Competitive Comparison
+
+| Metric | Smart Turn v3 | Krisp | Ultravox |
+|--------|---------------|-------|----------|
+| Size | 8 MB | 65 MB | 1.37 GB |
+| Languages | 23 | English only | 26 |
+| Availability | Open weights/data | Proprietary | Open weights |
+| Focus | Single-inference latency | Decision confidence | Conversation context |
+
+## Performance Benchmarks
+
+CPU inference results (including preprocessing):
+
+| Platform | Speed |
+|----------|-------|
+| AWS c7a.2xlarge | 12.6 ms |
+| AWS c8g.2xlarge | 15.2 ms |
+| Modal (6 cores) | 17.7 ms |
+| AWS t3.2xlarge | 33.8 ms |
+| AWS c8g.medium | 59.8 ms |
+| AWS t3.medium | 94.8 ms |
+
+GPU performance shows 3.3-6.6ms latency across various NVIDIA processors.
+
+## Accuracy Results
+
+Highest performers: Turkish (97.10%), Korean (96.85%), Japanese (96.76%)
+
+Lower performers: Vietnamese (81.27%), Bengali (84.10%), Marathi (87.60%)
+
+English achieved 94.31% accuracy across 2,846 test samples.
+
+## Implementation
+
+### With Pipecat
+`LocalSmartTurnAnalyzerV3` integration available (v0.0.85+). Users download ONNX model from HuggingFace repository.
+
+### Standalone
+Direct ONNX runtime usage via provided inference scripts. Requires accompanying VAD model (Silero recommended) for optimal results.
+
+## Resources
+
+- Model weights on HuggingFace
+- GitHub repository containing training code and inference examples
+- Open test datasets available for benchmark reproduction
+- Community dataset annotation available at smart-turn-dataset.pipecat.ai

03-finetune-pipecat-pt/references/blogs/pipecat_smart_turn_v3_1.md

@@ -0,0 +1,69 @@
+---
+title: "Improved Accuracy in Smart Turn v3.1"
+source: https://www.daily.co/blog/improved-accuracy-in-smart-turn-v3-1/
+date_accessed: 2026-03-16
+---
+
+# Improved Accuracy in Smart Turn v3.1
+
+Smart Turn v3.1 represents a significant advancement in conversation turn detection, leveraging expanded training datasets to enhance model performance across languages.
+
+## Overview
+
+Smart Turn v3.1 maintains the same architecture as its predecessor while offering improved accuracy through additional human audio training data and refined quantization methods. The update functions as a direct replacement for v3.0, requiring no code modifications to existing implementations.
+
+## Training Data Enhancement
+
+The model benefits from contributions by three specialized data partners:
+
+**Liva AI** provides "real human voice data to improve speech models" across multiple languages and dialects, founded by researchers with publications in IEEE and machine learning conferences.
+
+**Midcentury** develops "multimodal-native research" datasets spanning 12+ languages, emphasizing real-world performance scenarios.
+
+**MundoAI** constructs "the world's largest and highest quality multimodal datasets" across 16+ languages, prioritizing multilingual diversity.
+
+These partners contributed audio samples in English and Spanish, publicly released through HuggingFace as `smart-turn-data-v3.1-train` and `smart-turn-data-v3.1-test` datasets.
+
+## Model Variants
+
+Two deployment options accommodate different hardware configurations:
+
+- **CPU Model (8MB, int8 quantized)**: Lightweight variant enabling ~12ms CPU inference, matching v3.0 sizing
+- **GPU Model (32MB, unquantized)**: Enhanced variant for GPU deployment with ~1% accuracy improvement
+
+## Accuracy Improvements
+
+Testing on the new v3.1 dataset demonstrates substantial gains:
+
+| Language | v3.0 | v3.1 (8MB) | v3.1 (32MB) |
+|----------|------|-----------|-----------|
+| English | 88.3% | 94.7% | 95.6% |
+| Spanish | 86.7% | 90.1% | 91.0% |
+
+The model supports 23 total languages; the remaining 21 maintain parity with v3.0 performance.
+
+## Performance Benchmarks
+
+Single-inference latency across representative hardware:
+
+| Device | v3.1 (8MB) | v3.1 (32MB) | Preprocessing |
+|--------|-----------|-----------|---------------|
+| GPU (NVIDIA L40S) | 2 ms | 1 ms | 1 ms |
+| GPU (NVIDIA T4) | 5 ms | 4 ms | 2 ms |
+| CPU (AWS c7a.2xlarge) | 9 ms | 13 ms | 7 ms |
+| CPU (AWS c8g.2xlarge) | 20 ms | 32 ms | 9 ms |
+| CPU (AWS c7a.medium) | 37 ms | 73 ms | 7 ms |
+| CPU (AWS c8g.medium) | 57 ms | 159 ms | 9 ms |
+
+Performance optimization is achievable through environment variable configuration:
+
+```
+OMP_NUM_THREADS=1
+OMP_WAIT_POLICY="PASSIVE"
+```
+
+## Resources
+
+- Model weights: https://huggingface.co/pipecat-ai/smart-turn-v3
+- GitHub repository: https://github.com/pipecat-ai/smart-turn
+- Datasets: https://huggingface.co/pipecat-ai

03-finetune-pipecat-pt/references/blogs/pipecat_smart_turn_v3_2.md

@@ -0,0 +1,41 @@
+---
+title: "Smart Turn v3.2: Handling Noisy Environments and Short Responses"
+source: https://www.daily.co/blog/smart-turn-v3-2-handling-noisy-environments-and-short-responses/
+date_accessed: 2026-03-16
+---
+
+# Smart Turn v3.2: Better Accuracy for AI Voice Agents
+
+## Overview
+
+Smart Turn v3.2 represents an update to an open-source turn detection model designed to help AI voice agents identify when users finish speaking. The model listens to raw audio and determines optimal response timing -- preventing interruptions while eliminating unnecessary delays.
+
+## Key Improvements in v3.2
+
+### Short Utterances Enhancement
+The model now handles brief verbal responses significantly better. Single words like "yes" or "okay" are "miscategorized 40% less often according to our public benchmarks." Two changes enabled this improvement:
+
+- Introduction of a new dataset focused on short utterances (planned for expansion)
+- Resolution of a padding issue during training that was compromising accuracy
+
+### Background Noise Robustness
+The updated version performs better in real-world environments by incorporating realistic cafe and office noise into training and testing datasets, moving beyond studio-quality audio assumptions.
+
+## Technical Specifications
+
+**Model Variants:**
+- CPU version: 8MB
+- GPU version: 32MB
+
+Both serve as drop-in replacements for v3.1, maintaining compatibility with existing implementations.
+
+## Available Resources
+
+**Open-source components:**
+- Model weights: Available on HuggingFace
+- Training code: Published on GitHub
+- Datasets: Two new datasets released for training and testing purposes
+
+## Integration
+
+The model integrates with Pipecat through the `LocalSmartTurnAnalyzerV3` constructor, allowing immediate use via the `smart_turn_model_path` parameter.

03-finetune-pipecat-pt/references/blogs/speechmatics_semantic_turn.md

@@ -0,0 +1,103 @@
+---
+title: "How to Build Smarter Turn Detection for Voice AI"
+source: https://blog.speechmatics.com/semantic-turn-detection
+date_accessed: 2026-03-16
+---
+
+# How to Build Smarter Turn Detection for Voice AI
+
+**Published:** May 12, 2025 | **Author:** Aaron Ng, Machine Learning Engineer | **Read Time:** 13 minutes
+
+## Introduction
+
+Voice AI systems face a fundamental challenge: determining when users have finished speaking. Traditional approaches relying on fixed silence periods (like 500ms) create frustrating interruptions. This article explores how semantic understanding can improve turn detection.
+
+## The Problem with Traditional Voice Activity Detection
+
+VAD systems detect speech boundaries through audio patterns alone. As the article explains, "VAD only understands audio patterns. It knows _when_ there's speech and when there isn't. What it doesn't know is _why_ there's a pause."
+
+The example demonstrates this limitation: when a customer pauses to check information ("Sure it's 123 764... (pauses to check their notes)"), VAD incorrectly interprets the silence as turn completion and triggers an interruption.
+
+## Semantic Turn Detection Solution
+
+Rather than relying solely on silence detection, the proposed approach uses instruction-tuned Small Language Models (SLMs) to understand conversational context. These models contain fewer than 10 billion parameters, enabling fast local inference critical for voice interactions.
+
+### Key Benefits
+
+- **Reduced Latency**: Local SLM inference avoids the 500ms+ delays of external API calls
+- **Cost Savings**: Fewer false interruptions mean fewer unnecessary LLM API calls for responses that get discarded
+- **Better UX**: Systems recognize contextual pauses versus actual turn completion
+
+## Technical Implementation
+
+### Core Mechanism
+
+The approach monitors the probability of the `<|im_end|>` token -- a special ChatML marker indicating turn completion. When this token's probability is high, the model predicts the user has finished speaking.
+
+The article illustrates this with examples:
+- "Can I have two chicken McNuggets and" -> Low `<|im_end|>` probability (incomplete thought)
+- "I have a problem with my card" -> Higher `<|im_end|>` probability (complete thought)
+
+### Architecture Components
+
+**1. Message Tokenization**
+
+Messages are formatted using ChatML structure with special tokens (`<|im_start|>`, `<|im_end|>`, `<|im_sep|>`). The implementation removes the final `<|im_end|>` token since the model predicts its presence.
+
+**2. Model Inference**
+
+The code uses `AutoModelForCausalLM` to extract logits for the final token position, computing log-softmax probabilities across the vocabulary.
+
+**3. Token Probability Extraction**
+
+The system identifies the `<|im_end|>` token among top-k predictions (k=20) and converts log probabilities to standard probabilities using exponential transformation.
+
+### Code Example Structure
+
+The implementation includes an `EndOfTurnModel` class with:
+
+- `_convert_messages_to_chatml()`: Formats conversations in ChatML format
+- `get_next_token_logprobs()`: Performs local inference to retrieve next-token probabilities
+- `process_result()`: Extracts target token probabilities
+- `predict_eot_prob()`: Orchestrates the full pipeline returning probability scores
+
+**Model Details**: The implementation uses `SmolLM2-360M-Instruct` from Hugging Face, chosen for efficiency and CPU compatibility.
+
+## Configuration Parameters
+
+- **MAX_HISTORY**: 4 messages (recent context window)
+- **DEFAULT_THRESHOLD**: 0.03 (default probability threshold)
+- **Top-k consideration**: 20 tokens
+
+## Practical Considerations
+
+### Token Selection Strategy
+
+Beyond `<|im_end|>`, punctuation marks (periods, question marks, exclamation points) can signal thought completion. Hybrid approaches monitoring multiple tokens may improve accuracy but risk introducing noise.
+
+### Hybrid Approach with VAD
+
+The article recommends combining semantic detection with VAD:
+- VAD provides high recall detecting speech presence
+- Semantic turn detection improves precision to reduce false interruptions
+- Dynamic grace periods can adjust based on speaking patterns
+
+### Threshold Determination
+
+The default 0.03 threshold serves as a starting point. "The optimal threshold depends on your specific SLM and can vary significantly between models." Precision-focused optimization using representative test sets is recommended.
+
+### Multilingual Support
+
+Supporting multiple languages requires instruction-tuned SLMs trained on diverse multilingual data, accounting for varied grammar and conversational norms.
+
+### Fine-Tuning Opportunities
+
+While general SLMs provide solid conversational understanding, task-specific fine-tuning on domain conversational data can improve accuracy for specialized terminology or industry-specific use cases.
+
+## Future Directions
+
+The article acknowledges limitations: "True conversational understanding requires more than just reading text or listening for gaps -- it needs an audio-native model" that processes tone, cadence, hesitation, and emphasis directly from audio signals.
+
+## Resource Reference
+
+A complete implementation is available on GitHub at the repository referenced in the article.

03-finetune-pipecat-pt/references/guides/focal_loss_calibration_emnlp_2022.md

@@ -0,0 +1,209 @@
+---
+title: "Calibrating Imbalanced Classifiers with Focal Loss: An Empirical Study"
+source: https://aclanthology.org/2022.emnlp-industry.14/
+date: 2026-03-16
+---
+
+# Calibrating Imbalanced Classifiers with Focal Loss: An Empirical Study
+
+**Authors:** Cheng Wang, Jorge Balazs, Gyuri Szarvas, Patrick Ernst, Lahari Poddar, Pavel Danchenko (Amazon)
+
+**Venue:** EMNLP 2022 Industry Track, December 9-11, Abu Dhabi, UAE, pages 155-163
+
+**DOI:** 10.18653/v1/2022.emnlp-industry.14
+
+**PDF:** https://aclanthology.org/2022.emnlp-industry.14.pdf
+
+## Abstract
+
+Imbalanced data distribution is a practical and common challenge in building ML models in industry, where data usually exhibits long-tail distributions. For instance, in virtual AI Assistants (Google Assistant, Amazon Alexa, Apple Siri), the "play music" or "set timer" utterance is exposed to an order of magnitude more traffic than other skills. This can easily cause trained models to overfit to the majority classes, leading to model miscalibration. The uncalibrated models output unreliable (mostly overconfident) predictions, which are at high risk of affecting downstream decision-making systems. The authors empirically show the effectiveness of model training with focal loss in learning better calibrated models, as compared to standard cross-entropy loss. Better calibration enables better control of the precision-recall trade-off for trained models.
+
+## 1. Introduction
+
+Building ML models in industry faces practical challenges from imbalanced data distributions, particularly long-tail distributions, which make models overfit to majority data classes and lead to miscalibration -- the model-predicted probability fails to estimate the likelihood of true correctness and provides over- or under-confident predictions.
+
+The study focuses on **return reason code prediction** in customer service chatbots as a practical application of focal loss for calibration.
+
+### Contributions
+
+- Empirically examine the effectiveness of using focal loss in handling model miscalibration in a practical application setting
+- Show that good calibration is important to achieve a desired precision or recall target by tuning classification thresholds (standard cross-entropy loss is incapable of this due to skewed predicted probability distribution)
+- Demonstrate performance of calibrated models through a chatbot serving customers across three conversational bot use-cases
+
+## 2. Focal Loss Formulation
+
+Focal loss is defined as:
+
+```
+L_f = - sum_{i=1}^{N} (1 - p_{i,y_i})^gamma * log(p_{i,y_i})
+```
+
+where `p_{i,y_i}` is predicted probability of the i-th sample and `gamma` is a hyper-parameter typically set to `gamma = 1`.
+
+### Theoretical Interpretation
+
+Focal loss can be interpreted as a trade-off between minimizing KL divergence and maximizing entropy:
+
+```
+L_f >= KL(q || p) + H(q) - gamma * H(p)
+```
+
+The rationale: we learn a probability `p` to have a high value (confident) due to the KL term, but not too high (overconfident) due to the entropy regularization term.
+
+### Practical Advantages
+
+Compared to other calibration methods (temperature scaling, Bayesian methods, label smoothing, kernel-based methods), focal loss:
+- Neither increases computational overhead nor requires architectural modifications
+- Offers **in-training implicit calibration** (unlike temperature scaling which requires post-training calibration)
+
+## 3. Calibration Metrics
+
+### Reliability Diagrams
+
+Predictions are grouped into N interval bins; accuracy vs. confidence is computed per bin:
+
+```
+acc(b_n) = (1/I_n) * sum_i 1(y_hat_i = y_i)
+conf(b_n) = (1/I_n) * sum_i p_hat_i
+```
+
+A perfectly calibrated model has `acc(b_n) = conf(b_n)` for all n.
+
+### Expected Calibration Error (ECE)
+
+```
+ECE = (1/I) * sum_{n=1}^{N} I_n * |acc(b_n) - conf(b_n)|
+```
+
+### Maximum Calibration Error (MCE)
+
+```
+MCE = max_n |acc(b_n) - conf(b_n)|
+```
+
+Particularly important in high-risk applications. For a perfectly calibrated classifier, both ECE and MCE equal 0.
+
+## 4. Datasets and Implementation
+
+### Task
+
+Binary and multi-class return reason code prediction in an online retail store:
+- **Binary:** "item is defective or does not work" (Label 0) vs. OTHERS (Label 1)
+- **Multi-class:** 4 specific return reasons + OTHERS (5 total classes)
+
+Both datasets exhibit class imbalance with OTHERS as the most frequent class.
+
+### Model Architecture
+
+- 2 bidirectional LSTM layers + 2 dense layers
+- Embedding dimension: 1024
+- Hidden layer dimension: 128 (binary), 512 (multi-class)
+- Dropout: 0.1 (embedding), 0.2 (dense)
+- Optimizer: Adam
+- Framework: PyTorch
+
+### Golden Dataset
+
+- Binary model: 1,013 human-annotated samples
+- Multi-class model: 1,839 human-annotated samples
+- Data split: 8:1:1 (train/val/test)
+
+## 5. Results
+
+### Binary Reason Code Prediction (Table 1)
+
+| Metric | CE | FL1 | FL3 | FL5 | FL10 |
+|--------|------|------|------|------|------|
+| Accuracy | **0.836** | 0.824 | 0.831 | 0.834 | 0.816 |
+| Precision | **0.838** | 0.822 | 0.830 | 0.834 | 0.807 |
+| Recall | **0.823** | 0.814 | 0.821 | 0.823 | 0.805 |
+| F1 | **0.828** | 0.817 | 0.824 | 0.827 | 0.806 |
+| NLL | 2.159 | 1.438 | 0.608 | 0.258 | **0.178** |
+| ECE | 0.168 | 0.166 | 0.139 | 0.080 | **0.078** |
+| MCE | 0.720 | 0.730 | 0.236 | **0.134** | 0.143 |
+
+**Key finding:** CE achieves best predictive performance, but FL significantly outperforms on calibration metrics (NLL, ECE, MCE). Higher gamma yields better calibration with modest predictive performance loss.
+
+### Multi-Reason Code Prediction (Table 2)
+
+| Metric | CE | FL1 | FL5 |
+|--------|------|------|------|
+| Accuracy | 0.751 | **0.760** | 0.751 |
+| Precision | **0.814** | 0.807 | **0.814** |
+| Recall | **0.757** | 0.755 | **0.757** |
+| F1 | **0.764** | 0.760 | **0.764** |
+| NLL | 0.599 | 0.429 | **0.309** |
+| ECE | 0.037 | **0.023** | 0.037 |
+| MCE | 0.296 | **0.197** | 0.299 |
+
+### Reliability Diagrams
+
+- CE model: ECE = 16.78%, MCE = 72.00% (binary)
+- FL5 model: ECE = 7.98%, MCE = 13.40% (binary)
+- FL10 model: ECE = 7.76%, MCE = 14.34% (binary)
+
+As gamma increases from CE to FL10, probability distributions shift from "spiking" (overconfident, p close to 0 or 1) to flatter distributions (e.g., p = {0.6, 0.4}).
+
+### Precision-Recall Trade-Off
+
+CE model produces polarized probabilities, making it difficult to tune precision based on a given recall or vice versa. FL models learn better-distributed probabilities across [0, 1], enabling effective threshold-based precision/recall tuning.
+
+## 6. Deployment Results (Online A/B Test)
+
+Model deployed in three conversational chatbot use-cases with threshold=0.512 for 85% target precision:
+
+### Online Evaluation (Table 3) -- Relative Improvements
+
+| Application | Metric | Treatment vs. Control |
+|------------|--------|----------------------|
+| Use-case A | AR | +2.13% |
+| Use-case A | PRR | +3.18% |
+| Use-case A | 24RR | -0.65% |
+| Use-case B | AR | +2.10% |
+| Use-case B | PRR | +0.97% |
+| Use-case B | 24RR | -0.68% |
+| Use-case C | AR | +3.98% |
+| Use-case C | PRR | +12.85% |
+| Use-case C | 24RR | -1.02% |
+
+**Metrics:**
+- **AR (Automation Rate):** % contacts resolved without human involvement (higher = better)
+- **PRR (Positive Response Rate):** % positive customer responses to chatbot resolution (higher = better)
+- **24RR (Repeat Rate):** % customers contacting again within 24h for same issue (lower = better)
+
+### Intrinsic Evaluation
+
+- Precision on deployed model: 384/485 = 83.8% (aligns with offline 81.4%)
+- Negative predictive value: 194/200 = 97% for OTHERS class
+
+## 7. Key Takeaways
+
+1. Focal loss provides simple, effective in-training calibration via entropy regularization
+2. Higher gamma values improve calibration without significantly hurting predictive performance
+3. Well-calibrated models enable practical precision/recall threshold tuning that CE-trained models cannot achieve
+4. The discrimination-calibration trade-off is modest -- best model balances both (FL5 recommended)
+5. Better calibration directly translates to improved downstream application metrics
+
+## Citation
+
+```bibtex
+@inproceedings{wang-etal-2022-calibrating,
+    title = "Calibrating Imbalanced Classifiers with Focal Loss: An Empirical Study",
+    author = {Wang, Cheng and Balazs, Jorge and Szarvas, Gy{\"o}rgy and Ernst, Patrick and Poddar, Lahari and Danchenko, Pavel},
+    booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing: Industry Track",
+    month = dec,
+    year = "2022",
+    address = "Abu Dhabi, UAE",
+    publisher = "Association for Computational Linguistics",
+    pages = "155--163",
+    doi = "10.18653/v1/2022.emnlp-industry.14"
+}
+```
+
+## References (Selected)
+
+- Lin et al., 2017 -- Focal loss for dense object detection (original focal loss paper)
+- Mukhoti et al., 2020 -- Calibrating deep neural networks using focal loss (NeurIPS)
+- Guo et al., 2017 -- On calibration of modern neural networks (ICML)
+- Pereyra et al., 2017 -- Regularizing neural networks by penalizing confident output distributions
+- Naeini et al., 2015 -- Obtaining well calibrated probabilities using Bayesian binning (AAAI)

03-finetune-pipecat-pt/references/guides/livekit_turn_detector.md

@@ -0,0 +1,170 @@
+---
+title: "LiveKit Turn Detector Model Card"
+source: https://huggingface.co/livekit/turn-detector
+date: 2026-03-16
+---
+
+# LiveKit Turn Detector
+
+An open-weights language model for contextually-aware end-of-utterance (EOU) detection in voice AI applications. The model predicts whether a user has finished speaking based on the semantic content of their transcribed speech, providing a critical complement to voice activity detection (VAD) systems.
+
+> For installation, usage examples, and integration guides, see the [LiveKit documentation](https://docs.livekit.io/agents/logic/turns/turn-detector/).
+
+## Overview
+
+Traditional voice agents rely on voice activity detection (VAD) to determine when a user has finished speaking. VAD works by detecting the presence or absence of speech in an audio signal and applying a silence timer. While effective for detecting pauses, VAD lacks language understanding and frequently causes false positives.
+
+**Example:** A user who says *"I need to think about that for a moment..."* and then pauses will be interrupted by a VAD-only system, even though they clearly intend to continue.
+
+This model adds semantic understanding to the turn detection process by:
+- Analyzing transcribed text of conversations in real time
+- Predicting the probability that the user has completed their turn
+- Reducing unwanted interruptions when integrated with VAD
+- Handling structured data input effectively (addresses, phone numbers, email addresses, credit card numbers)
+
+## Model Variants
+
+**Multilingual** (recommended) and **English-only** (deprecated) are distributed as INT8 quantized ONNX models (`model_q8.onnx`) optimized for CPU inference.
+
+> **The English-only model (`EnglishModel`) is deprecated.** Use the **multilingual model (`MultilingualModel`)** for all new projects. The multilingual model provides better accuracy across all languages thanks to knowledge distillation from a larger teacher model and expanded training dataset.
+
+## How It Works
+
+The model operates on transcribed text from a speech-to-text (STT) system, not raw audio.
+
+### Process Flow
+
+1. **Input**: Recent conversation history (up to **6 turns**, truncated to **128 tokens**) is formatted using the [Qwen chat template](https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct) with `<|im_start|>` / `<|im_end|>` delimiters. The final user message is left _without_ the closing `<|im_end|>` token.
+
+2. **Prediction**: The model predicts the probability of the `<|im_end|>` token appearing next:
+   - **High probability** -> user has likely finished their utterance
+   - **Low probability** -> user is likely to continue
+
+3. **Thresholding**: Per-language thresholds (stored in `languages.json`) convert raw probability into a binary decision, tuned to balance responsiveness and accuracy for each supported language.
+
+4. **Integration with VAD**: Works alongside the [Silero VAD](https://docs.livekit.io/agents/logic/turns/vad/) plugin. VAD handles speech presence detection and interruption triggering, while this model provides the semantic signal for when to commit a turn.
+
+### Text Preprocessing
+
+**Multilingual variant** applies:
+- NFKC unicode normalization
+- Lowercasing
+- Punctuation removal (preserving apostrophes and hyphens)
+- Whitespace collapsing
+
+**English-only variant** passes raw transcribed text without normalization.
+
+## Architecture and Training
+
+### Base Model
+
+Both variants are fine-tuned from [Qwen/Qwen2.5-0.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct), selected for strong performance on this task while enabling low-latency CPU inference.
+
+**Model Specifications:**
+- **Parameters**: 0.1B
+- **Tensor Type**: F32
+- **Chat Template**: Qwen format
+
+### Knowledge Distillation
+
+A **Qwen2.5-7B-Instruct** model was first fine-tuned as a teacher on end-of-turn prediction. Its knowledge was then distilled into the 0.5B student model:
+- Distilled model approaches teacher-level accuracy
+- Maintains efficiency of smaller architecture
+- Converges after approximately 1,500 training steps
+
+### Training Data
+
+The training dataset is a mix of:
+
+- **Real call center transcripts** covering diverse conversational patterns
+- **Synthetic dialogues** emphasizing structured data input:
+  - Addresses
+  - Email addresses
+  - Phone numbers
+  - Credit card numbers
+- **Multi-format STT outputs** to handle provider variation (e.g., "forty two" vs. "42"), ensuring consistent predictions across different STT engines without runtime overhead
+
+*Note: Structured data enhancements were added only to the English training set, but performance improvements generalized across languages due to the multilingual knowledge encoded in Qwen2.5.*
+
+### Quantization
+
+The trained model is exported to ONNX format and quantized to INT8 (`model_q8.onnx`), enabling efficient CPU-only inference with ONNX Runtime.
+
+## Supported Languages
+
+The multilingual model supports **14 languages**. The model relies on the STT provider to report the detected language, which is then used to select the appropriate per-language threshold.
+
+**Supported:** English, Spanish, French, German, Italian, Portuguese, Dutch, Chinese, Japanese, Korean, Indonesian, Turkish, Russian, Hindi
+
+## Benchmarks
+
+### Detection Accuracy (Multilingual Variant)
+
+- **True positive** -- the model correctly identifies the user has finished speaking
+- **True negative** -- the model correctly identifies the user will continue speaking
+
+| Language | True Positive Rate | True Negative Rate |
+|----------|-------------------|-------------------|
+| Hindi | 99.4% | 96.3% |
+| Korean | 99.3% | 94.5% |
+| French | 99.3% | 88.9% |
+| Indonesian | 99.3% | 89.4% |
+| Japanese | 99.3% | 88.8% |
+| Dutch | 99.3% | 88.1% |
+| Russian | 99.3% | 88.0% |
+| German | 99.3% | 87.8% |
+| Portuguese | 99.4% | 87.4% |
+| Turkish | 99.3% | 87.3% |
+| English | 99.3% | 87.0% |
+| Chinese | 99.3% | 86.6% |
+| Spanish | 99.3% | 86.0% |
+| Italian | 99.3% | 85.1% |
+
+### Improvement Over Prior Version
+
+The multilingual v0.4.1 release achieved a **39.23% relative improvement** in handling structured inputs (emails, addresses, phone numbers, credit card numbers) compared to the prior version, reducing premature interruptions during data collection scenarios.
+
+## Usage
+
+The model is designed for use as a turn detection plugin within the [LiveKit Agents](https://github.com/livekit/agents) framework.
+
+For complete installation instructions, code examples (Python and Node.js), and configuration options, see the **[LiveKit turn detector plugin documentation](https://docs.livekit.io/agents/logic/turns/turn-detector/)**.
+
+For broader context on how turn detection fits into the voice pipeline -- including VAD configuration, interruption handling, and manual turn control -- see the **[Turns overview](https://docs.livekit.io/agents/logic/turns/)**.
+
+## Deployment Requirements
+
+- **Runtime**: CPU-only (no GPU required). Uses [ONNX Runtime](https://onnxruntime.ai/) with the `CPUExecutionProvider`.
+- **RAM**: <500 MB for the multilingual model
+- **Instance type**: Use compute-optimized instances (e.g., AWS c6i, c7i). Avoid burstable instances (e.g., AWS t3, t4g) to prevent inference timeouts from CPU credit exhaustion.
+- **LiveKit Cloud**: The model is deployed globally on LiveKit Cloud. Agents running there automatically use the optimized remote inference service with no local resource requirements.
+
+## Limitations
+
+- **Text-only input**: The model operates on STT-transcribed text and cannot incorporate prosodic cues such as pauses, intonation, or emphasis. Future versions may integrate multimodal audio features.
+- **STT dependency**: Prediction quality depends on the accuracy and output format of the upstream STT provider. Mismatches between training and deployment STT formats may degrade performance.
+- **Context window**: Limited to 128 tokens across a maximum of 6 conversation turns.
+- **Language coverage**: Currently supports 14 languages. Performance on unsupported languages is undefined.
+- **Realtime model compatibility**: Cannot be used with audio-native realtime models (e.g., OpenAI Realtime API) without adding a separate STT service, which incurs additional cost and latency.
+
+## License
+
+This model is released under the [LiveKit Model License](https://huggingface.co/livekit/turn-detector/blob/main/LICENSE).
+
+## Resources
+
+- **[Documentation](https://docs.livekit.io/agents/logic/turns/turn-detector/)**: Full plugin documentation, installation, and integration guide
+- **[Turns Overview](https://docs.livekit.io/agents/logic/turns/)**: How turn detection fits into the LiveKit Agents voice pipeline
+- **[Blog: Improved End-of-Turn Model](https://blog.livekit.io/improved-end-of-turn-model-cuts-voice-ai-interruptions-39/)**: Technical deep dive on multilingual distillation approach and benchmarks
+- **[Blog: Using a Transformer for Turn Detection](https://blog.livekit.io/using-a-transformer-to-improve-end-of-turn-detection/)**: Original blog post introducing the concept and architecture
+- **[Video: LiveKit Turn Detector](https://youtu.be/OZG0oZKctgw)**: Overview video demonstrating the plugin
+- **[GitHub: Plugin Source](https://github.com/livekit/agents/tree/main/livekit-plugins/livekit-plugins-turn-detector)**: Source code for the `livekit-plugins-turn-detector` package
+- **[PyPI](https://pypi.org/project/livekit-plugins-turn-detector/)** | **[npm](https://www.npmjs.com/package/@livekit/agents-plugin-livekit)**: Package registries
+
+---
+
+**Model Stats:**
+- Downloads last month: 240,507
+- Model size: 0.1B params
+- Format: Safetensors (INT8 quantized ONNX)
+- Base Model: Qwen/Qwen2.5-0.5B-Instruct

03-finetune-pipecat-pt/references/guides/pipecat_data_generation_guide.md

@@ -0,0 +1,33 @@
+---
+title: "Pipecat Smart Turn - Data Generation Contribution Guide"
+source: https://raw.githubusercontent.com/pipecat-ai/smart-turn/main/docs/data_generation_contribution_guide.md
+date: 2026-03-16
+---
+
+# Contributing Training Data to Smart Turn
+
+## Audio Format Requirements
+
+**FLAC is the preferred format** for training data, with lossy formats like MP3 or Opus discouraged. Audio should be **mono audio with a bit depth of 16 bits** and works optimally at 16kHz sample rates, though higher rates are acceptable.
+
+## File Organization
+
+Contributors have flexibility in naming and directory structure. The recommended approach uses unique identifiers for files combined with directory labels by language and completeness status (e.g., `eng/incomplete/b3799254-8d6c-11f0-a90e-e7e92780240b.flac`).
+
+## Audio Length and Variation
+
+Each file must contain **one speech sample, no longer than 16 seconds.** Variety in length is encouraged, ranging from single words to complex sentences.
+
+## Content Guidelines
+
+Samples should represent **a single turn in the conversation** with only one speaker per file. Speech should resemble interactions with voice assistants or customer service representatives. The documentation emphasizes avoiding sentence repetition and background noise while excluding real personally identifiable information.
+
+## Complete vs. Incomplete Classification
+
+Samples require binary labeling. "Complete" samples represent finished thoughts suitable for immediate response, while "Incomplete" samples suggest the speaker will continue, ending with filler words, connectives, or suggestive prosody. Critically, incomplete samples **must not be cut off in the middle of a word** but rather end with full words and approximately 200ms of silence.
+
+A 50:50 split between categories is recommended for unbiased training.
+
+## Licensing and Submission
+
+Contributors must own recordings and secure speaker consent for public release. Datasets are published via HuggingFace. Submissions can occur through various cloud storage methods, with GitHub issues as the contact point.

03-finetune-pipecat-pt/references/guides/pipecat_dataset_v3_2.md

@@ -0,0 +1,93 @@
+---
+title: "Pipecat Smart Turn Data v3.2 - Training Dataset Card"
+source: https://huggingface.co/datasets/pipecat-ai/smart-turn-data-v3.2-train
+date: 2026-03-16
+---
+
+# Smart Turn Data v3.2 Training Dataset
+
+The official training dataset for **Smart Turn v3.2**, hosted on Hugging Face Datasets.
+
+## Key Specifications
+
+| Attribute | Value |
+|-----------|-------|
+| **Organization** | Pipecat |
+| **Dataset Size** | 100K - 1M rows |
+| **Total Rows** | 270,946 |
+| **Download Size** | 41.4 GB |
+| **Parquet Size** | 41.4 GB |
+| **Format** | Parquet |
+| **Split** | train (271k rows) |
+| **Subset** | default (271k rows) |
+
+## Data Modalities
+
+- **Audio** - Audio samples with duration ranging from 0.36s to 32.6s
+- **Text** - Language and metadata fields
+
+## Supported Libraries
+
+- Datasets
+- Dask
+- Polars
+
+## Dataset Schema
+
+### Fields
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `audio` | AudioObject | Audio samples |
+| `id` | Text | UUID identifier (36 chars) |
+| `language` | Text | 23 language classes |
+| `endpoint_bool` | Boolean | End-of-turn indicator |
+| `midfiller` | Boolean | Mid-utterance filler detection |
+| `endfiller` | Boolean | End-of-utterance filler detection |
+| `synthetic` | Boolean | Synthetic data flag |
+| `dataset` | Text | Source dataset identifier (12 classes) |
+| `spoken_text` | Null | Skipped column |
+
+### Language Coverage
+
+English, Chinese (zho), Vietnamese, Finnish, Spanish, Bengali, Hindi, Japanese, Portuguese, Russian, Korean, German, French, Dutch, Arabic, Marathi, Turkish, Icelandic, Polish, Ukrainian, Italian, Indonesian, Norwegian
+
+### Dataset Sources
+
+- chirp3_1, chirp3_2
+- liva_1
+- midcentury_1
+- mundo_1
+- rime_2
+- orpheus_endfiller_1
+
+## Metadata (Croissant Format)
+
+The dataset uses **Croissant ML Commons** metadata schema (v1.1):
+
+```json
+{
+  "@context": "https://schema.org/",
+  "conformsTo": "http://mlcommons.org/croissant/1.1",
+  "@type": "sc:Dataset",
+  "name": "smart-turn-data-v3.2-train"
+}
+```
+
+## Contributors
+
+### Direct Contributors
+- The Pipecat team
+- [Liva AI](https://www.theliva.ai/)
+- [Midcentury](https://www.midcentury.xyz/)
+- [MundoAI](https://mundoai.world/)
+
+### Background Noise Attribution
+CC-0 licensed background noise samples sourced from Freesound.org contributors including:
+- 4team, tomhannen, craigsmith, mrmayo, martats, and 26+ additional contributors
+
+## Access Methods
+
+- **Dataset Viewer:** https://huggingface.co/datasets/pipecat-ai/smart-turn-data-v3.2-train/viewer/
+- **Data Studio:** https://huggingface.co/datasets/pipecat-ai/smart-turn-data-v3.2-train/viewer/default/train
+- **Files Browser:** Git repository with parquet conversion available

03-finetune-pipecat-pt/references/guides/pipecat_smart_turn_readme.md

@@ -0,0 +1,115 @@
+---
+title: "Pipecat Smart Turn v3.2 - README"
+source: https://raw.githubusercontent.com/pipecat-ai/smart-turn/main/README.md
+date: 2026-03-16
+---
+
+# Smart Turn v3.2
+
+An open-source, community-driven native audio turn detection model designed to improve upon traditional voice activity detection (VAD) approaches. The model determines when a voice agent should respond to human speech by analyzing linguistic and acoustic cues rather than simply detecting speech presence.
+
+## Key Features
+
+**Language Support:** The system supports 23 languages including Arabic, Bengali, Chinese, Danish, Dutch, German, English, Finnish, French, Hindi, Indonesian, Italian, Japanese, Korean, Marathi, Norwegian, Polish, Portuguese, Russian, Spanish, Turkish, Ukrainian, and Vietnamese.
+
+**Performance Characteristics:**
+- Inference completes in as little as 10ms on certain CPUs
+- Most cloud instances see sub-100ms execution times
+- Integrates efficiently with lightweight VAD models like Silero
+- Two versions available: 8MB quantized CPU variant and 32MB unquantized GPU variant
+- GPU version uses fp32 weights for marginally faster inference and ~1% accuracy improvement
+- CPU version uses int8 quantization for reduced size and speed with minimal accuracy trade-off
+
+**Technical Approach:** The model operates directly on PCM audio samples rather than text transcriptions, capturing prosodic nuances that inform turn-taking decisions.
+
+## Setup Instructions
+
+**Environment Configuration:**
+```bash
+python3.12 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+```
+
+**Platform-Specific Dependencies:**
+
+Ubuntu/Debian systems require:
+```bash
+sudo apt-get update
+sudo apt-get install portaudio19-dev python3-dev
+```
+
+macOS with Homebrew:
+```bash
+brew install portaudio
+```
+
+**Running the Demonstration:**
+```bash
+python record_and_predict.py
+```
+
+Initial startup requires approximately 30 seconds. Test phrases include "I can't seem to, um ..." and "I can't seem to, um, find the return label."
+
+## Audio Input Specifications
+
+The model accepts 16kHz mono PCM audio with maximum duration of 8 seconds. The recommended approach involves providing the complete audio of the current user turn. When audio exceeds 8 seconds, truncate from the beginning while maintaining context.
+
+For shorter audio, prepend zero-value padding to reach the required length, ensuring actual speech content occupies the end of the input vector.
+
+## Model Architecture
+
+Smart Turn v3 employs Whisper Tiny as its foundation with an added linear classification layer. The transformer-based architecture contains approximately 8 million parameters. Development involved experimentation with wav2vec2-BERT, wav2vec2, LSTM implementations, and additional transformer classifier configurations.
+
+## Integration Methods
+
+**Pipecat Integration:** The framework supports local inference via `LocalSmartTurnAnalyzerV3` (version 0.0.85 and later). On Pipecat Cloud's standard 1x instance, inference typically completes in around 65ms.
+
+**Direct Integration:** Import `model.py` and `inference.py` from the Smart Turn repository and invoke the `predict_endpoint()` function. Reference implementation available in `predict.py`.
+
+## Training Infrastructure
+
+Training code resides in `train.py` and downloads datasets from the pipecat-ai HuggingFace repository. Training can execute locally or via Modal using `train_modal.py`. Training sessions log to Weights & Biases unless disabled.
+
+**Training command for Modal:**
+```bash
+modal run --detach train_modal.py
+```
+
+**Current Datasets:**
+- pipecat-ai/smart-turn-data-v3.2-train
+- pipecat-ai/smart-turn-data-v3.2-test
+
+## Community Contributions
+
+**Data Classification:** Manual training data categorization assistance needed at https://smart-turn-dataset.pipecat.ai/
+
+**Human Data Contribution:** Participants can contribute through turn training games at https://turn-training.pipecat.ai/ or by following the data generation contribution guide.
+
+**Future Development Areas:**
+- Additional language support expansion
+- Performance optimization and architecture refinement
+- Expanded human dataset collection
+- Text-conditioned inference for specialized input modes
+- Training platform diversification
+
+## Licensing and Attribution
+
+Smart Turn operates under the BSD 2-clause license, permitting unrestricted usage, modification, and contribution.
+
+**Project Contributors:**
+- Marcus (marcus-daily)
+- Eli (ebb351)
+- Mark (markbackman)
+- Kwindla (kwindla)
+
+**Data Contributors:**
+- Liva AI
+- Midcentury
+- MundoAI
+
+## Resources
+
+- [HuggingFace Model Repository](https://huggingface.co/pipecat-ai/smart-turn-v3)
+- [Pipecat Documentation](https://docs.pipecat.ai/server/utilities/smart-turn/smart-turn-overview)
+- [Pipecat Framework](https://pipecat.ai)

03-finetune-pipecat-pt/references/guides/pipecat_train_py.md

@@ -0,0 +1,179 @@
+---
+title: "Pipecat Smart Turn - Training Code (train.py)"
+source: https://raw.githubusercontent.com/pipecat-ai/smart-turn/main/train.py
+date: 2026-03-16
+---
+
+# SmartTurn V3 Speech Endpointing Model — Training Code Reference
+
+> This document describes the training pipeline implemented in `train.py` from the
+> [pipecat-ai/smart-turn](https://github.com/pipecat-ai/smart-turn) repository.
+> The original file is Python source code; key architecture and configuration details
+> are extracted below.
+
+## Core Architecture
+
+### SmartTurnV3Model
+
+The model extends `WhisperPreTrainedModel` and uses Whisper's encoder as its backbone:
+
+- **Input**: Log-mel spectrogram features with shape `(batch_size, 80, 800)`
+- **Encoder**: Modified Whisper encoder with `max_source_positions=400`
+- **Attention Pooling**: Neural network layer that learns weighted attention across time steps
+- **Binary Classifier**: Multi-layer sequential network outputting probability scores
+
+```
+Input Features (batch, 80, 800)
+    |
+Whisper Encoder
+    |
+Attention Pooling [Linear -> Tanh -> Linear -> Softmax]
+    |
+Weighted Pooling (reduces to batch_size, hidden_size)
+    |
+Classifier [Linear -> LayerNorm -> GELU -> Dropout -> Linear -> GELU -> Linear]
+    |
+Sigmoid Output (probability of completion)
+```
+
+### Loss Function
+
+Uses `BCEWithLogitsLoss` with dynamic positive sample weighting, clamped between 0.1 and 10.0 to handle class imbalance.
+
+## Training Configuration
+
+```
+Base Model:                openai/whisper-tiny
+Learning Rate:             5e-5
+Epochs:                    4
+Train Batch Size:          384
+Eval Batch Size:           128
+Warmup Ratio:              0.2
+Weight Decay:              0.01
+Eval/Save Steps:           500
+Logging Steps:             100
+LR Scheduler:              Cosine
+Dataloader Workers:        6
+```
+
+## Datasets
+
+**Training Sources**:
+- `pipecat-ai/smart-turn-data-v3.2-train` (split 90/10 for train/eval)
+
+**Test Sources**:
+- `pipecat-ai/smart-turn-data-v3.2-test`
+
+The system supports stratified analysis by language, midfiller presence, and endfiller presence.
+
+## Data Pipeline
+
+### OnDemandSmartTurnDataset
+
+On-demand feature extraction pipeline:
+- Audio truncation to last 8 seconds
+- 16kHz sampling rate
+- Padding to max length: `8 * 16000 = 128,000 samples`
+- Normalization enabled
+
+### SmartTurnDataCollator
+
+Batches samples while preserving metadata (language, midfiller, endfiller flags) for downstream analysis.
+
+## Export and Quantization
+
+### ONNX FP32 Export
+
+The wrapper ensures consistent output shapes across variable batch sizes:
+- Dynamic batch dimension
+- Fixed output shape: `(batch_size, 1)` for compatibility
+- ONNX opset version 18
+- Validation with batch sizes 1 and 2
+
+### INT8 Static Quantization
+
+**Configuration**:
+- Calibration dataset size: 1024 samples
+- Quantization format: QDQ (Quantize-Dequantize)
+- Activation type: QUInt8
+- Weight type: QInt8
+- Per-channel quantization enabled
+- Calibration method: Entropy
+- Quantized operations: Conv, MatMul, Gemm
+
+Process:
+1. `quant_pre_process` for optimization and shape inference
+2. Entropy-based calibration on training data subset
+3. Static quantization with calibration reader
+
+## Evaluation Metrics
+
+Per-sample metrics computed during training and evaluation:
+
+- Accuracy
+- Precision (with zero_division warning handling)
+- Recall
+- F1 Score
+- Confusion matrix components (TP, FP, TN, FN)
+- Predicted positive/negative counts
+
+### External Evaluation Callback
+
+During training checkpoints, the system evaluates on all test splits and logs:
+- Dataset-specific accuracies
+- Language-stratified metrics
+- Midfiller-stratified metrics
+- Probability distributions (Weights & Biases histograms)
+- Min/max/mean accuracy across categories
+- Sample count distributions
+
+## Key Utility Functions
+
+- **`truncate_audio_to_last_n_seconds`**: Crops audio to preserve final n seconds (prevents padding inflation)
+- **`process_predictions`**: Converts logits to probabilities and binary predictions using 0.5 threshold, with NaN validation
+- **`compute_metrics`**: Scikit-learn based metric computation with detailed confusion matrix breakdown
+
+## Workflows
+
+### Training Run (`do_training_run`)
+
+1. Initialize Weights & Biases project "speech-endpointing"
+2. Load pretrained Whisper-tiny with custom head
+3. Prepare datasets (load, split, wrap)
+4. Train with specified hyperparameters
+5. Save final model and feature extractor
+6. Export to ONNX FP32
+7. Return export path
+
+### Quantization Run (`do_quantization_run`)
+
+1. Load FP32 ONNX model
+2. Create calibration dataset from training split
+3. Apply pre-processing (optimization)
+4. Execute static INT8 quantization
+5. Return quantized model path
+
+### Benchmark Run (`do_benchmark_run`)
+
+1. Load feature extractor
+2. Prepare test merged dataset
+3. For each model path:
+   - Create benchmark output directory
+   - Run benchmark with batch size 256
+   - Generate markdown report
+
+## Dependencies
+
+- PyTorch with ONNX export capabilities
+- Transformers (HuggingFace) for Whisper and training
+- ONNX Runtime with quantization support
+- Scikit-learn for metrics
+- Weights & Biases for experiment tracking
+- torchcodec (runtime requirement for audio decoding)
+
+## Error Handling
+
+- Fast-fail check for missing `torchcodec` dependency
+- ONNX model validation after export
+- Non-finite value detection in predictions
+- Exception logging for failed exports

03-finetune-pipecat-pt/references/guides/vogent_turn_80m.md

@@ -0,0 +1,171 @@
+---
+title: "Vogent-Turn-80M Model Card"
+source: https://huggingface.co/vogent/Vogent-Turn-80M
+date: 2026-03-16
+---
+
+# Vogent-Turn-80M
+
+A state-of-the-art multimodal turn detection model for voice AI systems, achieving **94.1% accuracy** by combining acoustic and linguistic signals for real-time conversational applications.
+
+## Overview
+
+| Attribute | Value |
+|-----------|-------|
+| **Developed by** | Vogent AI |
+| **Model type** | Multimodal Turn Detection (Binary Classification) |
+| **Language(s)** | English |
+| **License** | Modified Apache-2.0 (horizontal voice agent platforms cannot set as default) |
+| **Base model** | SmolLM2-135M (reduced to 80M parameters using first 12 layers) |
+| **Model size** | 79.2M parameters (F32) |
+| **Inference speed** | ~7ms on T4 GPU |
+
+## Model Description
+
+Vogent-Turn-80M determines when a speaker has finished their turn in a conversation by processing both:
+- **Acoustic features** (via Whisper encoder)
+- **Semantic context** (via language model)
+
+This multimodal approach enables real-time, accurate predictions where audio-only or text-only methods fail.
+
+### Resources
+
+- **GitHub Repository:** https://github.com/vogent/vogent-turn
+- **Technical Report:** https://blog.vogent.ai/posts/voturn-80m-state-of-the-art-turn-detection-for-voice-agents
+- **Demo Space:** https://huggingface.co/spaces/vogent/vogent-turn-demo
+
+## Quick Install
+
+```bash
+git clone https://github.com/vogent/vogent-turn.git
+cd vogent-turn
+pip install -e .
+```
+
+## Basic Usage
+
+```python
+from vogent_turn import TurnDetector
+import soundfile as sf
+import urllib.request
+
+# Initialize detector
+detector = TurnDetector(compile_model=True, warmup=True)
+
+# Download and load audio
+audio_url = "https://storage.googleapis.com/voturn-sample-recordings/incomplete_number_sample.wav"
+urllib.request.urlretrieve(audio_url, "sample.wav")
+audio, sr = sf.read("sample.wav")
+
+# Run turn detection with conversational context
+result = detector.predict(
+    audio,
+    prev_line="What is your phone number",
+    curr_line="My number is 804",
+    sample_rate=sr,
+    return_probs=True,
+)
+
+print(f"Turn complete: {result['is_endpoint']}")
+print(f"Done speaking probability: {result['prob_endpoint']:.1%}")
+```
+
+## Available Interfaces
+
+- **Python Library:** Direct integration with `TurnDetector` class
+- **CLI Tool:** `vogent-turn-predict speech.wav --prev "What is your phone number" --curr "My number is 804"`
+
+## Technical Architecture
+
+### Model Architecture
+
+**Components:**
+1. **Audio Encoder:** Whisper-Tiny (processes up to 8 seconds of 16kHz audio)
+2. **Text Model:** SmolLM-135M (12 layers, ~80M parameters)
+3. **Multimodal Fusion:** Audio embeddings projected into LLM's input space
+4. **Classifier:** Binary classification head (turn complete/incomplete)
+
+### Processing Flow
+
+```
+1. Audio (16kHz PCM) -> Whisper Encoder -> Audio Embeddings (~400 tokens)
+2. Text Context -> SmolLM Tokenizer -> Text Embeddings
+3. Concatenate embeddings -> SmolLM Transformer -> Last token hidden state
+4. Linear Classifier -> Softmax -> [P(continue), P(endpoint)]
+```
+
+## Training Details
+
+### Preprocessing
+
+- **Audio:** Last 8 seconds extracted via Whisper-Tiny encoder -> ~400 audio tokens
+- **Text:** Full conversational context (assistant and user utterances)
+- **Labels:** Binary classification (turn complete/incomplete)
+- **Fusion:** Audio embeddings projected into LLM's input space and concatenated with text
+
+### Training Hyperparameters
+
+- **Training regime:** fp16 mixed precision
+- **Base model:** SmolLM2-135M (first 12 layers)
+- **Architecture:** Reduced from 135M to ~80M parameters via layer ablation
+
+### Training Data
+
+Diverse dataset combining:
+- Human-collected conversational data
+- Synthetic conversational data
+
+## Evaluation Results
+
+### Performance Metrics
+
+- **Accuracy:** 94.1%
+- **AUPRC:** 0.975
+
+### Testing Data
+
+Internal test set covering diverse conversational scenarios and edge cases where audio-only or text-only approaches fail.
+
+## Compute Infrastructure
+
+### Hardware Optimization
+
+- `torch.compile` with max-autotune mode
+- Dynamic tensor shapes without recompilation
+- Pre-warmed bucket sizes (64, 128, 256, 512, 1024)
+
+### Software
+
+- **Framework:** PyTorch with torch.compile
+- **Audio processing:** Whisper encoder (up to 8 seconds)
+
+## Limitations
+
+- **English-only** support; turn-taking conventions vary across languages and cultures
+- **CPU inference** may be too slow for some real-time applications
+
+## Citation
+
+```bibtex
+@misc{voturn2025,
+  title={Vogent-Turn-80M: State-of-the-Art Turn Detection for Voice Agents},
+  author={Varadarajan, Vignesh and Vytheeswaran, Jagath},
+  year={2025},
+  publisher={Vogent AI},
+  howpublished={\url{https://huggingface.co/vogent/Vogent-Turn-80M}},
+  note={Blog: \url{https://blog.vogent.ai/posts/voturn-80m-state-of-the-art-turn-detection-for-voice-agents}}
+}
+```
+
+## Additional Resources
+
+- **Full Documentation & Code:** https://github.com/vogent/vogent-turn
+- **Platform:** https://vogent.ai
+- **Enterprise Contact:** j@vogent.ai
+- **Issues:** https://github.com/vogent/vogent-turn/issues
+
+## Upcoming Releases
+
+- Int8 quantized model for faster CPU deployment
+- Multilingual versions
+- Domain-specific adaptations

03-finetune-pipecat-pt/references/hesitation_turn_taking_l2_review.md

@@ -0,0 +1,453 @@
+# Pausas de Hesitacao, Deteccao de Fim de Turno e Fala L2: Uma Revisao para Fine-Tuning em Portugues
+
+**Contexto**: BabelCast — traducao simultanea em reunioes. O modelo de deteccao de fim de turno precisa distinguir pausas de hesitacao (falante ainda vai continuar) de fim de turno real (pode comecar a traduzir). O desafio e especialmente critico para falantes de frances aprendendo portugues, que produzem pausas longas de hesitacao frequentemente confundidas com fim de turno.
+
+**Autores**: Marcos Remar, com assistencia de Claude (Anthropic)
+**Data**: Março 2026
+
+---
+
+## 1. O Problema: Silencio Nao Significa Fim de Turno
+
+Sistemas comerciais de IA conversacional usam thresholds de silencio entre 700ms e 1000ms para detectar fim de turno (Castillo-Lopez, de Chalendar & Semmar, 2025). Esse approach e fundamentalmente inadequado por dois motivos:
+
+1. **Humanos sao mais rapidos**: o gap medio entre turnos em conversacao natural e de apenas ~200ms (Levinson & Torreira, 2015, citado em Skantze, 2021). Esperar 700ms+ resulta em respostas percebidas como lentas.
+
+2. **Pausas dentro de turnos sao frequentemente mais longas que gaps entre turnos** (Skantze, 2021). Um falante pode pausar 2-3 segundos no meio de uma frase (buscando uma palavra ou planejando o proximo trecho) e continuar normalmente. Tratar essa pausa como fim de turno causa interrupcao.
+
+O problema se agrava dramaticamente quando o falante esta usando uma segunda lingua (L2), como demonstrado na literatura a seguir.
+
+---
+
+## 2. Pausas de Hesitacao em Falantes L2
+
+### 2.1 Duracao: L2 pausa 39% mais que nativos
+
+Kosmala & Crible (2022) analisaram pausas preenchidas (filled pauses, FPs) em falantes nativos e nao-nativos de frances, usando o corpus SITAF:
+
+| Metrica | Nativos | Nao-nativos | Diferenca |
+|---------|---------|-------------|-----------|
+| Duracao media FP | 378ms (SD=200) | **524ms** (SD=222) | +146ms (+39%) |
+| Taxa FP/100 palavras | 4.4 | 5.3 | +0.9 (n.s.) |
+| Clustering com outros fluencemas | 72% | **82%** | +10 p.p. |
+
+A diferenca de **frequencia** nao e significativa — nao-nativos nao produzem mais pausas, mas pausas **significativamente mais longas** (p < .001). A duracao e um indicador mais confiavel de proficiencia que a frequencia.
+
+Em situacoes de reparo (self-repair), as pausas sao ainda mais extremas. Kosmala (2025) analisou 167 reparos de alunos franceses falando ingles e encontrou:
+
+- Pausa silenciosa media na fase de edicao: **844ms** (SD=573ms)
+- Pausa preenchida media na fase de edicao: **522ms** (SD=571ms)
+- 82% dos auto-reparos contem uma fase de edicao (pausa e/ou filler)
+
+Esses 844ms estao **muito acima** de qualquer threshold de silencio usado em sistemas comerciais (700ms).
+
+### 2.2 Tipos de pausa: silenciosas dominam
+
+Cenoz (2000) estudou 15 falantes nativos de espanhol falando ingles como L2, analisando 1.085 pausas nao-juntivas:
+
+| Tipo | Proporcao | Faixa de duracao |
+|------|-----------|------------------|
+| Silenciosas | **64%** | 205ms a 11.569ms |
+| Preenchidas | 36% | — |
+
+Distribuicao por duracao das pausas silenciosas:
+- 70% entre 200ms e 1.000ms
+- 21% entre 1.001ms e 2.000ms
+- 7% entre 2.001ms e 4.000ms
+- 2% acima de 4.000ms
+
+**Implicacao critica**: a maioria das hesitacoes sao **silencio puro** — nao contem fillers como "euh" ou "hum" que o modelo poderia usar como pista. Isso torna a deteccao mais dificil, pois o modelo precisa distinguir silencio de hesitacao de silencio de fim de turno usando apenas prosodia.
+
+### 2.3 Funcao das pausas: planejamento vs busca lexical
+
+Cenoz (2000) classificou as funcoes das pausas:
+
+| Funcao | Pausas silenciosas | Pausas preenchidas |
+|--------|-------------------|-------------------|
+| Planejamento (frases, sintaxe) | 59% | **73%** |
+| Busca lexical (palavras) | 36% | 26% |
+| Outros | 5% | 1% |
+
+Pausas preenchidas sao usadas primariamente para **manter o turno** (floor-holding): 77% ocorrem isoladas, sem outros marcadores de hesitacao. Ja pausas silenciosas co-ocorrem com outras estrategias de reparo 54% das vezes.
+
+### 2.4 Paradoxo de proficiencia
+
+Um achado contraintuitivo: falantes **mais proficientes** produzem **mais** pausas preenchidas, nao menos (Cenoz, 2000). Na faixa de maior proficiencia, a proporcao muda para 53% silenciosas + 46% preenchidas (vs. 69%/31% nos menos proficientes). Isso ocorre porque falantes avancados aprenderam a usar fillers como estrategia de floor-holding — sinalizam ao interlocutor que ainda estao falando.
+
+A variacao individual e enorme: a proporcao de pausas preenchidas varia de **4% a 74.5%** entre falantes individuais.
+
+---
+
+## 3. Fillers Franceses na Fala L2
+
+### 3.1 Transferencia linguistica
+
+Falantes franceses transferem seus fillers nativos para a L2. Lo (2018) analisou 15 bilingues alemao-frances e mostrou que as propriedades acusticas dos fillers sao distintas por lingua:
+
+- **Frances "euh"**: duracao mais longa, F1-F3 mais altos (vogal central schwa)
+- **Alemao "ah"**: duracao mais curta, F1-F3 mais baixos (vogal aberta)
+
+Bilingues mantem formas foneticamente distintas para cada lingua, indicando que **a forma do filler revela qual lingua esta sendo processada**. Isso e confirmado por Kosmala (2025): alunos franceses falando ingles usam "euh" (24 instancias) e "eum" (12 instancias), os fillers franceses, nao os ingleses.
+
+### 3.2 Fillers sao itens lexicais, nao sons universais
+
+Bottcher & Zellers (2024) analisaram o corpus RUEG (736 falantes, 5 linguas, 4.468 narracoes) e demonstraram que fillers sao **itens lexicais especificos de cada lingua**, nao sons universais de hesitacao. A proporcao nasal ("uhm") vs. nao-nasal ("uh") varia por lingua, genero e idade.
+
+Falantes heritage (bilingues desde a infancia) produzem **mais fillers no total** devido a carga cognitiva da ativacao de duas linguas. A **tolerancia ao silencio** tambem se transfere da L1: falantes de japones mantem maior tolerancia ao silencio mesmo falando ingles.
+
+### 3.3 Code-switching: 93% nas fronteiras de unidade
+
+Beatty-Martinez, Navarro-Torres & Dussias (2020) analisaram 10 bilingues espanhol-ingles e encontraram que **93% das trocas de codigo ocorrem em fronteiras de unidades entonacionais** — os mesmos pontos onde transicoes de turno acontecem. Antes de uma troca, ha **reorganizacao prosodica** (mudanca na velocidade de fala), que pode ser confundida com marcadores prosodicos de fim de turno.
+
+**Implicacao para o modelo**: quando um frances falando portugues alterna para uma palavra em frances ("Eu preciso de... *comment dit-on*... uma tesoura"), a mudanca prosodica antes da troca NAO indica fim de turno.
+
+---
+
+## 4. Pistas Prosodicas de Fim de Turno
+
+### 4.1 O contorno de F0 e o sinal-chave
+
+Ekstedt & Skantze (2022) conduziram experimentos de perturbacao prosodica com o modelo VAP para isolar a importancia relativa de cada pista:
+
+| Pista prosodica | Importancia |
+|----------------|-------------|
+| Informacao fonetica/espectral | Mais importante no geral |
+| **Contorno de F0 (pitch)** | **Mais importante para desambiguar pontos sintaticamente completos** |
+| Intensidade | Comparavel ao F0 no geral |
+| Normalizacao de duracao | Menos importante individualmente |
+
+O achado central: **F0 e o principal desambiguador em pontos onde a sintaxe poderia indicar completude**. Um F0 ascendente + silaba final mais longa = sinal de completude de turno. O modelo VAP consegue prever trocas de turno **antes** da completude do enunciado a partir da dinamica prosodica.
+
+### 4.2 Prosodia antes de pausas preenchidas
+
+Wu, Didirkova & Simon (2023) analisaram o corpus LOCAS-F (2h38m, >1.000 FPs, kappa inter-anotador = 0.86) e encontraram padroes prosodicos especificos antes de fillers:
+
+- Queda de F0 de **1-2 semitons** durante pausas preenchidas (17.90 vs. 19.01 ST)
+- Reset melodico **negativo significativo** antes da FP (descontinuidade de pitch)
+- Efeito de preparacao: queda de **4.66 ST** entre fala nao-preparada (18.89 ST) e preparada (14.24 ST)
+- **Nao ha reset significativo** entre a FP e a silaba seguinte
+
+**Implicacao**: a queda de pitch antes de um filler "euh" e diferente da queda de pitch de fim de frase. O modelo precisa aprender essa distincao sutil — o que o encoder do Whisper pode capturar atraves de representacoes espectrais.
+
+### 4.3 Resumo das pistas prosodicas de turno
+
+Skantze (2021) compilou a literatura:
+
+| Pista | Fim de turno | Pausa de hesitacao |
+|-------|-------------|-------------------|
+| Pitch (F0) | Queda (declarativa) ou queda-ascensao | Suspenso ou queda menor |
+| Intensidade | Diminui gradualmente | Mantem-se ou cai abruptamente |
+| Velocidade de fala | Desacelera antes do fim | Pode acelerar antes de parar |
+| Alongamento silabico | Silaba final alongada | Ausente |
+| Completude sintatica | Frase sintaticamente completa | Frase incompleta |
+
+---
+
+## 5. Estado da Arte em Deteccao de Fim de Turno
+
+### 5.1 Modelos e suas acuracias
+
+| Modelo | Tipo | Params | Tamanho | Latencia | Acuracia | Linguas |
+|--------|------|--------|---------|----------|----------|---------|
+| **Pipecat Smart Turn v3.1** | Audio (Whisper Tiny) | 8M | 8 MB (INT8) | 12ms CPU | 94.7% (EN) | 23 |
+| **LiveKit v0.4.1** | Texto (Qwen2.5-0.5B) | 500M | — | — | TNR 87.4% (PT) | 15+ |
+| **Vogent-Turn-80M** | Multimodal | 79.2M | — | 7ms T4 | 94.1% (EN) | 1 |
+| **Krisp TT v1** | Audio | 6.1M | 65 MB | — | 82% bal.acc. | 1 |
+| **VAP** | Audio (CPC+Transformer) | — | — | 14.6ms CPU | 76.2% bal.acc. | 3 |
+| **SpeculativeETD** | Audio (GRU+Wav2vec) | 1M+94M | — | 0.26ms+server | 66% real | 1 |
+
+Nenhum destes modelos foi especificamente otimizado para portugues ou para fala L2.
+
+### 5.2 Pipecat Smart Turn: resultados por lingua
+
+| Lingua | Acuracia | FP | FN |
+|--------|----------|-----|-----|
+| Turco | 97.10% | 1.66% | 1.24% |
+| Coreano | 96.85% | 1.12% | 2.02% |
+| Japones | 96.76% | 2.04% | 1.20% |
+| Frances | 96.01% | 1.60% | 2.39% |
+| **Portugues** | **95.42%** | **2.79%** | **1.79%** |
+| Ingles | 94.31% | 2.64% | 3.06% |
+| Espanhol | 91.97% | 4.48% | 3.55% |
+
+Portugues esta atras de frances, japones, coreano e turco. A taxa de falsos positivos (2.79%) indica que o modelo diz "terminou" quando nao terminou em quase 3% dos casos — problematico para traducao simultanea.
+
+### 5.3 O gap sintetico → real: o maior risco
+
+Ok, Yoo & Lee (2025) demonstraram um colapso devastador quando modelos treinados em dados sinteticos (TTS) sao avaliados em fala real:
+
+| Modelo | F1 sintetico | F1 real | Perda |
+|--------|-------------|---------|-------|
+| Wav2vec 2.0 | 94.7% | **30.3%** | -64.4 p.p. |
+| SpeculativeETD | 88.9 IoU | **16.4 IoU** | -72.5 p.p. |
+| VAP | 87.7 IoU | **10.7 IoU** | -77.0 p.p. |
+
+**Todos os modelos colapsam** ao sair de dados sinteticos para conversacao real. A melhoria de v3.0 para v3.1 do Pipecat veio justamente de adicionar **audio humano real** de tres parceiros especializados (Liva AI, Midcentury, MundoAI).
+
+---
+
+## 6. Tecnicas de Treinamento Validadas
+
+### 6.1 Focal Loss (Lin et al., 2017)
+
+A Focal Loss resolve o desbalanceamento extremo de classes (a maioria dos frames e "fala em andamento"; fim de turno e raro):
+
+```
+FL(p_t) = -α_t · (1 - p_t)^γ · log(p_t)
+```
+
+Com gamma=2, a perda para exemplos bem classificados (p_t=0.9) e **100x menor** que cross-entropy padrao; com p_t=0.968, **1.000x menor**. Os melhores hiperparametros: **gamma=2, alpha=0.25** (Lin et al., 2017, Tabela 1a).
+
+### 6.2 Knowledge Distillation (LiveKit)
+
+O LiveKit treinou um modelo professor (Qwen2.5-7B) e destilou para um aluno (Qwen2.5-0.5B), que converge apos ~1.500 steps. O resultado: reducao de 39% nos falsos positivos de interrupcao. Para portugues especificamente, a melhoria relativa foi de **45.97%** — a segunda maior entre todas as linguas.
+
+### 6.3 Dados curtos e ruidosos (Pipecat v3.2)
+
+O Pipecat v3.2 adicionou dois tipos de dados ao treinamento:
+1. **Respostas curtas** ("yes", "no", "ok"): reduziu erros de classificacao em **40%**
+2. **Ruido de fundo** (cafe, escritorio, CC-0 Freesound): melhorou robustez
+
+### 6.4 Injecao de fillers e pausas (SpeculativeETD)
+
+Ok et al. (2025) testaram tres variantes de dados sinteticos:
+- V1: TTS direto (baseline)
+- V2: Pausas de hesitacao estendidas para 1.5-3.0 segundos
+- **V3: Fillers injetados ("um", "uh") em posicoes aleatorias + pausas apos**
+
+V3 foi a melhor variante, validando a abordagem de gerar dados com fillers inseridos por LLM.
+
+### 6.5 Transfer learning cross-lingual
+
+Castillo-Lopez et al. (2025) documentam que VAP pre-treinado em ingles (Switchboard) e fine-tunado em japones **supera** o treinamento direto em japones. Isso valida a abordagem de partir do Pipecat (pre-treinado em 23 linguas) e fine-tunar para portugues.
+
+---
+
+## 7. Pesquisa: Modelos de Turn-Taking para Aprendizado de Idiomas (Marco 2026)
+
+**Data da pesquisa**: 16 de marco de 2026
+**Objetivo**: identificar modelos e tecnicas especificas para turn-taking em contexto de aprendizado de idiomas com avatar conversacional
+
+### 7.1 Panorama: nenhum modelo especifico existe
+
+Nao existe nenhum modelo open-source de turn-taking treinado especificamente para aprendizes L2. Os produtos comerciais usam abordagens genericas:
+
+| Produto | Abordagem | Limitacao |
+|---------|-----------|-----------|
+| **Praktika** (OpenAI) | GPT-5.2 Realtime API — turn detection nativo OpenAI | Caixa preta, sem adaptacao L2 |
+| **ConversAR** (Meta Quest) | Timeout fixo 2000ms + "periodo infinito de pensamento" | Nao distingue hesitacao de fim de turno |
+| **Gliglish** | ChatGPT + speech recognition generico | Sem turn detection especializado |
+| **ELSA Speak** | Modelo proprietario treinado com sotaques variados | Foco em pronuncia, nao turn-taking |
+| **TalkPal/SpeakPal/Talkio** | GPT + NLP generico | Sem modelo de audio para end-of-turn |
+| **Hume EVI** | Prosodia (tom de voz) para turn detection | Comercial, sem foco em L2 |
+| **Deepgram Flux** | Modelo fusionado (ASR + turn detection) com `eot_threshold` configuravel | So ingles, sem adaptacao para proficiencia |
+
+### 7.2 Modelos de pesquisa relevantes
+
+- **VAP (Voice Activity Projection)** — multilingual (EN/ZH/JA), fine-tuning por lingua melhora significativamente, mas nao cobre L2 ou non-native speakers (Inoue et al., 2024)
+- **Speak & Improve Corpus 2025** — 340 horas de fala L2 ingles com anotacao de disfluencias e CEFR scores (A2-C1, maioria B1-B2) (Knill et al., 2025)
+- **Whisper + LoRA para hesitation tagging** — fine-tune do Whisper Large V3 com tags de hesitacao acusticamente precisas: **11.3% melhoria no WER** para fala L2 (arXiv:2506.04076)
+- **Deepgram Flux** — modelo fusionado (ASR + turn detection em um unico modelo); ~260ms end-of-turn detection, ~30% menos interrupcoes vs pipeline tradicional (Deepgram, 2025)
+- **Survey IWSDS 2025** — 72% dos trabalhos de turn-taking NAO comparam com metodos anteriores, sugerindo falta de benchmarks estabelecidos (Castillo-Lopez et al., 2025)
+
+### 7.3 Tecnicas identificadas para melhorar o fine-tuning
+
+**1. Threshold duplo (Eager + Final) — inspirado no Deepgram Flux**
+
+Em vez de um unico threshold, usar dois:
+- **Eager threshold (0.3-0.5)**: o avatar comeca a preparar resposta especulativamente (inicia geracao LLM)
+- **Final threshold (0.7+)**: confirma fim de turno e fala
+
+Economia de latencia estimada: 150-250ms no inicio da resposta, sem interromper o falante.
+
+Deepgram Flux implementa isso como `eot_threshold` (padrao 0.7) e `eager_eot_threshold` (0.3-0.5), com parametro `eot_timeout_ms` para configurar sensibilidade.
+
+**2. Custo assimetrico (FP >> FN)**
+
+Para um tutor de idiomas, **interromper o aluno e muito pior que esperar demais**:
+- ConversAR (2025): "periodo infinito de pensamento" — learners controlam quando falar
+- Praktika: timeout estendido para fala L2 fragmentada e com sotaque
+- Implementacao: `fp_penalty=2.0` no focal loss — erros de interrupcao custam 2x mais
+
+**3. Dados L2 reais — Speak & Improve Corpus**
+
+O corpus Speak & Improve 2025 (Knill et al., 2025) tem:
+- 340 horas de audio L2 ingles
+- Anotacao de disfluencias (false starts, repeticoes, hesitacoes)
+- Scores CEFR: B1 (18.3%), B1+ (25.3%), B2 (25.1%), B2+ (18.3%)
+- Embora seja ingles L2, padroes de hesitacao L2 sao transferiveis entre linguas (Cenoz, 2000)
+
+Fine-tuning do Whisper com anotacao precisa de hesitacoes (tags "um"/"uh" acusticamente alinhadas) mostrou 11.3% melhoria no WER vs ignorar hesitacoes.
+
+**4. Proficiency-aware turn-taking (futuro)**
+
+Nenhum sistema implementa isso ainda, mas e logico:
+- Aluno A1: pausa **3-5x mais** que B2
+- Aluno B2: usa fillers como estrategia de floor-holding (Cenoz, 2000)
+- De Jong & Bosker: threshold otimo de 250ms para pausa L2 em holandes
+- Shea & Leonard (2019): threshold de **1000ms** para espanhol L2
+- Implementacao futura: input de nivel CEFR ao modelo, ou adaptar threshold baseado em perfil do falante
+
+### 7.5 Sistema de backchannel para avatar de aprendizado
+
+Um problema critico identificado: se o avatar espera 3 segundos em silencio, o aprendiz pensa que o sistema travou e para de falar. A solucao e um sistema de **backchannel signals** que mostra ao aprendiz que o avatar esta ouvindo, sem tomar o turno.
+
+| Tempo de silencio | Acao do avatar | Tipo |
+|-------------------|----------------|------|
+| 0-600ms | Nada (normal) | — |
+| 600ms-1.5s | Aceno de cabeca, olhar atento | Backchannel visual |
+| 1.5s-3.0s | "Mhm...", "Continue...", "Uhum..." | Backchannel verbal |
+| 3.0s+ | "Sem pressa, pode pensar...", "Prenez votre temps..." | Encorajamento |
+
+O sistema integra o threshold duplo do modelo com os backchannels:
+- **Eager threshold** atingido → avatar da backchannel + inicia geracao LLM especulativa
+- **Final threshold** atingido → avatar fala a resposta
+
+Presets por nivel CEFR:
+
+| CEFR | Eager | Final | BC Visual | BC Verbal | Encorajamento |
+|------|-------|-------|-----------|-----------|---------------|
+| A1 | 0.50 | **0.80** | 500ms | 1200ms | 2500ms |
+| A2 | 0.45 | 0.75 | 550ms | 1300ms | 2800ms |
+| B1 | 0.40 | 0.70 | 600ms | 1500ms | 3000ms |
+| B2 | 0.35 | 0.65 | 600ms | 1800ms | 4000ms |
+| C1 | 0.35 | **0.60** | 600ms | 2000ms | 5000ms |
+
+O avatar e mais paciente com A1/A2 (threshold final 0.80 = raramente interrompe) e mais responsivo com B2/C1 (0.60-0.65 = conversa mais natural). Implementado em `06_inference.py`.
+
+### 7.6 Conclusao da pesquisa
+
+O trabalho do BabelCast e confirmado como **pioneiro**: nao existe modelo de turn-taking para aprendizes L2, muito menos para francofonos falando portugues. As melhorias implementadas (custo assimetrico, threshold duplo, dados L2 reais, backchannel por CEFR) trazem o modelo mais proximo do comportamento ideal de um tutor paciente.
+
+---
+
+## 8. Implicacoes para o Fine-Tuning BabelCast
+
+### 8.1 O que o modelo precisa aprender
+
+Com base na literatura, os seguintes padroes sao criticos para o modelo distinguir:
+
+| Padrao | Duracao tipica | Label | Pista prosodica |
+|--------|---------------|-------|----------------|
+| Fim de turno real | silencio 200-500ms | COMPLETO | F0 caindo, intensidade caindo, silaba final alongada |
+| Hesitacao nativa (PT-BR) | FP 378ms + silencio 200-1000ms | INCOMPLETO | F0 suspenso, "hum"/"tipo"/"ne" |
+| Hesitacao L2 (francofono) | FP **524ms** + silencio **844ms** | INCOMPLETO | F0 suspenso, "euh"/"alors"/"comment dire" |
+| Busca de palavra L2 | silencio 1000-3000ms | INCOMPLETO | Silencio puro, sem pista prosodica clara |
+| Code-switching | prosodic reset + silencio 500-1500ms | INCOMPLETO | Mudanca de velocidade antes da troca |
+| Resposta curta completa | "Sim." + silencio 200ms | COMPLETO | F0 caindo, curta duracao |
+| Resposta curta incompleta | "Sim, mas..." + silencio | INCOMPLETO | F0 suspenso ou ascendente |
+
+### 8.2 Dados necessarios
+
+1. **Audio real de portugues** (CORAA MUPE 365h, NURC-SP 239h) — evitar o gap sintetico→real
+2. **TTS com fillers** brasileiros ("hum", "tipo", "ne") e franceses ("euh", "alors") inseridos por LLM
+3. **Pausas longas de hesitacao** (1.5-3.0s) injetadas em amostras incompletas
+4. **Respostas curtas** ("sim", "nao", "ok", "tá bom") com variantes completas e incompletas
+5. **Code-switching** frances-portugues em fronteiras de unidade
+6. **Ruido de fundo** real (cafe, escritorio) — CC-0 Freesound
+
+### 8.3 Escolhas arquiteturais validadas
+
+- **Whisper Tiny encoder** (39M params, 8s janela, 384-dim): mesma arquitetura do Pipecat, captura prosodia sem decodificar texto
+- **Attention pooling**: aprende quais frames perto do silencio sao mais informativos (Ekstedt & Skantze, 2022)
+- **Focal Loss** (gamma=2, alpha=0.25): calibracao implícita sem label smoothing (EMNLP 2022)
+- **INT8 quantizacao**: 32MB → 8MB, 12ms CPU (Pipecat deploy pipeline)
+
+### 8.4 Lacuna na literatura
+
+**Nenhum paper foi encontrado especificamente sobre deteccao de fim de turno em fala L2**, e especialmente nao sobre francofonos falando portugues. A intersecao de:
+- Deteccao de fim de turno (ML/audio)
+- Pausas de hesitacao em L2 (linguistica)
+- Fala francofona em portugues (fonologia)
+
+...e uma area completamente inexplorada. O trabalho do BabelCast e, ate onde sabemos, o primeiro a atacar esse problema especifico.
+
+---
+
+## Referencias
+
+### Papers Academicos
+
+1. Beatty-Martinez, A. L., Navarro-Torres, C. A., & Dussias, P. E. (2020). Codeswitching: A bilingual toolkit for opportunistic speech planning. *Frontiers in Psychology*, 11, 1699.
+
+2. Bottcher, A. & Zellers, M. (2024). Do you say uh or uhm? A cross-linguistic approach to filler particle use in heritage and majority speakers across three languages. *Frontiers in Psychology*, 15, 1358182.
+
+3. Castillo-Lopez, V., de Chalendar, G., & Semmar, N. (2025). A survey of recent advances on turn-taking modeling in spoken dialogue systems. *arXiv:2503.xxxxx*.
+
+4. Cenoz, J. (2000). Pauses and hesitation phenomena in second language production. *ITL - International Journal of Applied Linguistics*, 127-128, 53-69.
+
+5. Christodoulides, G. & Avanzi, M. (2014). DisMo: A morphosyntactic, disfluency and multi-word unit annotator. *Proceedings of LREC 2014*.
+
+6. Christodoulides, G. & Avanzi, M. (2015). Automatic detection and annotation of disfluencies in spoken French corpora. *Proceedings of Interspeech 2015*.
+
+7. Ekstedt, E. & Skantze, G. (2022). How much does prosody help turn-taking? Investigations using voice activity projection models. *Proceedings of Interspeech 2022*.
+
+8. Inoue, K., Jiang, D., Ekstedt, E., Kawahara, T., & Skantze, G. (2024). Real-time and continuous turn-taking prediction using voice activity projection. *arXiv:2401.04868*.
+
+9. Inoue, K., Jiang, D., Ekstedt, E., Kawahara, T., & Skantze, G. (2024). Multilingual turn-taking prediction using voice activity projection. *Proceedings of LREC-COLING 2024*.
+
+10. Kosmala, L. & Crible, L. (2022). The dual status of filled pauses: Evidence from genre, proficiency and co-occurrence. *Language and Speech*, 65(4), 1-25.
+
+11. Kosmala, L. (2023). Exploring the status of filled pauses as pragmatic markers: The role of gaze and gesture. *Journal of Pragmatics*, 212, 1-15.
+
+12. Kosmala, L. (2025). Multimodal self- and other-initiated repairs in L2 peer interactions. *Proceedings of DiSS 2025*, Lisbon.
+
+13. Lin, T.-Y., Goyal, P., Girshick, R., He, K., & Dollar, P. (2017). Focal loss for dense object detection. *Proceedings of ICCV 2017*. arXiv:1708.02002.
+
+14. Lo, S. L. (2018). Between ah(m) and euh(m): Filled pauses in German-French bilinguals. *BAAP 2018 Poster Presentation*.
+
+15. Ok, J., Yoo, I. C., & Lee, Y. (2025). Speculative end-turn detector: Revisiting an efficient design for low-latency end-of-turn detection. *arXiv:2503.23439*.
+
+16. Peters, M. (2017). L2 fluency development in French. *Ph.D. Thesis*.
+
+17. Raux, A. & Eskenazi, M. (2009). A finite-state turn-taking model for spoken dialog systems. *Proceedings of NAACL-HLT 2009*.
+
+18. Skantze, G. (2021). Turn-taking in conversational systems and human-robot interaction: A review. *Computer Speech & Language*, 67, 101178.
+
+19. Wu, X., Didirkova, I., & Simon, A. C. (2023). Disfluencies in continuous speech in French: Prosodic parameters of filled pauses and vowel lengthening. *Proceedings of ICPhS 2023*.
+
+20. Knill, K., et al. (2025). Speak & Improve Corpus 2025: an L2 English Speech Corpus for Language Assessment and Feedback. *arXiv:2412.11986*.
+
+21. Saeki, T., et al. (2025). Acoustically precise hesitation tagging is essential for end-to-end verbatim transcription systems. *arXiv:2506.04076*.
+
+22. Gamboa, H. & Wohlgenannt, G. (2025). ConversAR: Practicing a second language without fear — Mixed reality agents for interactive group conversation. *arXiv:2510.08227*.
+
+23. De Jong, N. & Bosker, H. R. (2013). Choosing a threshold for silent pauses to measure second language fluency. *The 6th Workshop on Disfluency in Spontaneous Speech (DiSS)*.
+
+24. Shea, C. & Leonard, K. (2019). Evaluating measures of pausing for second language fluency research. *Journal of Second Language Pronunciation*, 5(2), 254-277.
+
+### Modelos e Blogs Tecnicos
+
+20. Daily/Pipecat. (2025). Announcing Smart Turn v3, with CPU inference in just 12ms. https://www.daily.co/blog/announcing-smart-turn-v3-with-cpu-inference-in-just-12ms/
+
+21. Daily/Pipecat. (2025). Improved accuracy in Smart Turn v3.1. https://www.daily.co/blog/improved-accuracy-in-smart-turn-v3-1/
+
+22. Daily/Pipecat. (2026). Smart Turn v3.2: Handling noisy environments and short responses. https://www.daily.co/blog/smart-turn-v3-2-handling-noisy-environments-and-short-responses/
+
+23. LiveKit. (2025). Improved end-of-turn model cuts voice AI interruptions 39%. https://livekit.com/blog/improved-end-of-turn-model-cuts-voice-ai-interruptions-39/
+
+24. LiveKit. (2025). Using a transformer to improve end-of-turn detection. https://blog.livekit.io/using-a-transformer-to-improve-end-of-turn-detection
+
+25. Krisp. (2025). Audio-only 6M weights turn-taking model for voice AI agents. https://krisp.ai/blog/turn-taking-for-voice-ai/
+
+26. Deepgram. (2025). Evaluating end-of-turn detection models. https://deepgram.com/learn/evaluating-end-of-turn-detection-models
+
+27. Vogent. (2025). Vogent-Turn-80M model card. https://huggingface.co/vogent/Vogent-Turn-80M
+
+28. Deepgram. (2025). Introducing Flux: Conversational Speech Recognition. https://deepgram.com/learn/introducing-flux-conversational-speech-recognition
+
+29. Hume AI. (2025). Empathic Voice Interface (EVI) documentation. https://dev.hume.ai/docs/speech-to-speech-evi/overview
+
+30. OpenAI. (2025). Inside Praktika's conversational approach to language learning. https://openai.com/index/praktika/
+
+31. Tavus. (2025). The Complete Guide to AI Turn-Taking. https://www.tavus.io/post/ai-turn-taking
+
+### Datasets
+
+32. Pipecat-AI. (2026). Smart Turn Data v3.2 Training Set. HuggingFace: `pipecat-ai/smart-turn-data-v3.2-train`. 270,946 samples, 23 languages.
+
+33. NILC-NLP. CORAA-MUPE-ASR. HuggingFace: `nilc-nlp/CORAA-MUPE-ASR`. 365h, Portuguese interviews.
+
+34. NILC-NLP. CORAA NURC-SP Audio Corpus. HuggingFace: `nilc-nlp/CORAA-NURC-SP-Audio-Corpus`. 239h, Portuguese dialogues.
+
+35. Cambridge English. Speak & Improve Corpus 2025. 340h, L2 English learner speech with disfluency annotations and CEFR scores.

03-finetune-pipecat-pt/references/papers/finite_state_turn_taking_raux_2009.md

@@ -0,0 +1,158 @@
+---
+title: "A Finite-State Turn-Taking Model for Spoken Dialog Systems"
+authors:
+  - Antoine Raux
+  - Maxine Eskenazi
+year: 2009
+source: https://aclanthology.org/N09-1071/
+date_converted: 2026-03-16
+---
+
+## Abstract
+
+This paper introduces the Finite-State Turn-Taking Machine (FSTTM), a new model to control the turn-taking behavior of conversational agents. Based on a non-deterministic finite-state machine, the FSTTM uses a cost matrix and decision-theoretic principles to select a turn-taking action at any time. The authors show how the model can be applied to the problem of end-of-turn detection. Evaluation results on a deployed spoken dialog system show that the FSTTM provides significantly higher responsiveness than previous approaches.
+
+**Note**: The PDF file `finite_state_turn_taking_raux_2009.pdf` in this directory is corrupted (contains an unrelated physics paper). Content for this summary was sourced from the actual paper PDF downloaded from ACL Anthology.
+
+## Key Contributions
+
+1. **FSTTM model**: A principled, unified framework for turn-taking control based on a 6-state non-deterministic finite-state machine with decision-theoretic action selection.
+2. **Data-driven optimization**: Unlike prior hand-coded models, the FSTTM's cost parameters can be optimized from data using logistic regression.
+3. **Anytime endpointing**: The model can make end-of-turn decisions both during pauses and during speech (before a pause is even detected by VAD), enabling faster response times.
+4. **Deployed evaluation**: Tested on a real, publicly deployed bus information system, not just in lab conditions.
+
+## Architecture / Method Details
+
+### Six-State Finite-State Model
+
+Based on Jaffe & Feldstein (1970) and Brady (1969), the FSTTM uses six states representing who holds the conversational floor:
+
+| State | Description |
+|-------|-------------|
+| **USER** | User has the floor (obligation or intention to speak) |
+| **SYSTEM** | System has the floor |
+| **FREES** | Floor is free, following a SYSTEM state |
+| **FREEU** | Floor is free, following a USER state |
+| **BOTHS** | Both claim the floor, following a SYSTEM state |
+| **BOTHU** | Both claim the floor, following a USER state |
+
+States are defined in terms of **intentions and obligations**, not surface-level speech/silence observations. For example, USER state persists during mid-turn pauses.
+
+### Four Turn-Taking Actions
+
+At any time, the system can take one of four actions:
+- **Grab (G)**: Claim the floor
+- **Release (R)**: Relinquish the floor
+- **Keep (K)**: Maintain floor claim
+- **Wait (W)**: Remain silent without claiming floor
+
+### Turn-Taking Phenomena Captured
+
+The model formalizes common turn-taking patterns as 2-step state transitions:
+
+1. **Turn transitions with gap**: `SYSTEM --(R,W)--> FREES --(W,G)--> USER` (most common)
+2. **Turn transitions with overlap**: `SYSTEM --(K,G)--> BOTHS --(R,K)--> USER` (barge-in)
+3. **Failed interruptions**: `USER --(G,K)--> BOTHU --(R,K)--> USER` (system interrupts then backs off)
+4. **Time outs**: `SYSTEM --(R,W)--> FREES --(G,W)--> SYSTEM` (user doesn't respond)
+
+### Decision-Theoretic Action Selection
+
+The optimal action minimizes expected cost:
+
+```
+C(A) = sum over S in States: P(s=S|O) * C(A, S)
+```
+
+Where `P(s=S|O)` is estimated from observable features and `C(A,S)` comes from a cost matrix.
+
+### Cost Matrix Design Principles
+
+Derived from Sacks et al. (1974) -- "participants minimize gaps and overlaps":
+
+1. Actions that resolve gaps/overlaps have zero cost.
+2. Actions that create unwanted gaps/overlaps have a constant cost parameter.
+3. Actions that maintain gaps/overlaps have cost proportional to time in that state.
+
+Four cost parameters:
+- **CS**: Cost of false interruption (interrupting system prompt when user is not claiming floor)
+- **CO(tau)**: Cost of remaining in overlap for tau ms
+- **CU**: Cost of cut-in (grabbing floor when user holds it)
+- **CG(tau)**: Cost of remaining in a gap for tau ms (set as CG^p * tau)
+
+### Probability Estimation
+
+The key estimation task is `P(FREEU | Ot)` -- the probability that the user has released the floor.
+
+**At pauses** (VAD detects silence): Uses Bayes rule combining:
+- `P(F|O)`: Prior probability of floor release from pause-onset features (logistic regression)
+- `P(d >= tau | O, U)`: Probability pause lasts at least tau ms given user holds floor (exponential distribution)
+
+**During speech** (before VAD pause detection): Separate logistic regression on each ASR partial hypothesis, enabling endpointing before a full pause is detected.
+
+### Features Used
+
+All features are automatically extractable at runtime:
+- **Dialog state**: Open question / Closed question / Confirmation
+- **Turn-taking features**: Whether current utterance is a barge-in
+- **Semantic features**: From dialog state and partial ASR hypotheses
+- **Boundary LM score**: Ratio of log-likelihood of hypothesis being complete vs. incomplete (trained on ASR output, no human transcription needed). **Most informative feature across all states.**
+- **Average words per utterance** so far
+- **Confirmation markers**: "YES", "SURE", etc.
+- **Pause duration** within partial hypothesis (0-200ms range)
+
+## Experimental Results
+
+### Logistic Regression for P(F|O)
+
+Trained with stepwise regression, 10-fold cross-validation on 586 dialogs (2008 corpus):
+
+| Dialog State | Pause: Class. Error | Pause: Log-Likelihood | Speech: Class. Error | Speech: Log-Likelihood |
+|---|---|---|---|---|
+| Open question | 35% (vs 38% baseline) | -0.61 (vs -0.66) | 17% (vs 20%) | -0.40 (vs -0.50) |
+| Closed question | 26% (vs 25% baseline) | -0.50 (vs -0.56) | 22% (vs 32%) | -0.49 (vs -0.63) |
+| Confirmation | 12% (vs 12% baseline) | -0.30 (vs -0.36) | 17% (vs 36%) | -0.40 (vs -0.65) |
+
+The "in speech" model achieves much larger gains, especially for Closed questions and Confirmations -- classification error drops from 32% to 22% and 36% to 17% respectively.
+
+### Batch Evaluation: Latency vs. Cut-in Rate
+
+**In-pause FSTTM vs. baselines** (2007 corpus):
+- FSTTM outperforms fixed threshold baseline by up to **29.5% latency reduction**.
+- Slight improvement over Ferrer et al. (2003) reimplementation.
+
+**Anytime-FSTTM vs. in-pause FSTTM** (2008 corpus):
+- At 5% cut-in rate: anytime-FSTTM yields latencies **17% shorter** than in-pause-FSTTM, and **40% shorter** than fixed threshold baseline.
+- **30-40% of turns are endpointed before the pause is detected by VAD** (during speech).
+
+### Live Evaluation (Deployed System)
+
+A/B test over 10 days: 171 FSTTM dialogs vs. 148 fixed-threshold control dialogs.
+
+Settings: `CG^p = 1, CG^s = 500, CU = 5000` (FSTTM) vs. 555ms fixed threshold (control). Both calibrated for ~6.3% cut-in rate.
+
+| Metric | FSTTM | Fixed Threshold |
+|--------|-------|-----------------|
+| Average latency | **320ms** | 513ms |
+| Cut-in rate | **4.8%** | 6.3% |
+
+Results:
+- **193ms latency reduction** (p < 0.05, statistically significant).
+- 1.5% cut-in rate reduction (not statistically significant, but directionally correct).
+
+## Relevance to Turn-Taking / End-of-Turn Detection
+
+The FSTTM is a foundational reference for BabelCast's end-of-turn detection:
+
+1. **Decision-theoretic framework**: The cost-based approach directly applies to our system. We can define costs for premature LLM invocation (wasted compute, incorrect partial translation) vs. delayed response (poor user experience), and optimize the trade-off.
+
+2. **Anytime endpointing**: The insight that 30-40% of turns can be endpointed *before the pause is detected by VAD* is powerful. In our pipeline, this means we could start the translation process before the speaker's pause is even fully formed, dramatically reducing perceived latency.
+
+3. **Boundary LM score**: The most informative feature in Raux's system was a language model score measuring syntactic completeness. We already have ASR partial results from Whisper -- we could compute a similar feature to predict turn completeness without waiting for silence.
+
+4. **Cost parameter tuning**: The four-parameter cost matrix (CS, CO, CU, CG) provides a compact, interpretable way to tune system behavior. We could expose these as configuration parameters in our Pipecat pipeline, allowing adjustment of responsiveness vs. interruption rate.
+
+5. **Real-world validation**: This is one of the few ETD papers validated on a deployed system with real users, not just lab data. The 193ms latency improvement is a concrete, practically meaningful result.
+
+6. **Simplicity**: The FSTTM is mathematically elegant and computationally trivial -- just a few probability estimates and a cost comparison. It could be implemented alongside our Silero VAD as an additional decision layer with negligible overhead.
+
+7. **Limitation for our case**: The system was designed for a task-oriented telephone dialog (bus information), which has more predictable turn structures than the open-domain meeting conversations we handle. The boundary LM approach would need adaptation for our more varied content.

03-finetune-pipecat-pt/references/papers/finite_state_turn_taking_raux_2009.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaa1054cf8f25ccdfd847cd0a323e8d0473867404c5895551a1ec9889e4f7d97
+size 254541

03-finetune-pipecat-pt/references/papers/focal_loss_lin_2017.md

@@ -0,0 +1,103 @@
+---
+title: "Focal Loss for Dense Object Detection"
+authors:
+  - Tsung-Yi Lin
+  - Priya Goyal
+  - Ross Girshick
+  - Kaiming He
+  - Piotr Dollar
+year: 2017
+source: https://arxiv.org/abs/1708.02002
+date_converted: 2026-03-16
+---
+
+## Abstract
+
+The highest accuracy object detectors to date are based on a two-stage approach popularized by R-CNN, where a classifier is applied to a sparse set of candidate object locations. In contrast, one-stage detectors that are applied over a regular, dense sampling of possible object locations have the potential to be faster and simpler, but have trailed the accuracy of two-stage detectors thus far. The authors discover that the extreme foreground-background class imbalance encountered during training of dense detectors is the central cause. They propose to address this class imbalance by reshaping the standard cross entropy loss such that it down-weights the loss assigned to well-classified examples. The novel Focal Loss focuses training on a sparse set of hard examples and prevents the vast number of easy negatives from overwhelming the detector during training. Using focal loss, their one-stage RetinaNet detector matches the speed of previous one-stage detectors while surpassing the accuracy of all existing state-of-the-art two-stage detectors.
+
+## Key Contributions
+
+1. **Focal Loss function**: A dynamically scaled cross entropy loss where the scaling factor decays to zero as confidence in the correct class increases, automatically down-weighting easy examples and focusing on hard examples.
+2. **RetinaNet**: A simple one-stage dense object detector built on Feature Pyramid Networks (FPN) that, when trained with focal loss, achieves state-of-the-art results.
+3. **Identification of class imbalance** as the central obstacle preventing one-stage detectors from matching two-stage detector accuracy.
+
+## Method Details
+
+### Focal Loss Definition
+
+Standard cross entropy loss:
+
+```
+CE(pt) = -log(pt)
+```
+
+Focal loss adds a modulating factor `(1 - pt)^gamma`:
+
+```
+FL(pt) = -alpha_t * (1 - pt)^gamma * log(pt)
+```
+
+Key properties:
+- When `gamma = 0`, FL is equivalent to CE.
+- When an example is misclassified (`pt` is small), the modulating factor is near 1 and the loss is unaffected.
+- As `pt -> 1`, the factor goes to 0, down-weighting well-classified examples.
+- With `gamma = 2`, an example classified with `pt = 0.9` has 100x lower loss compared to CE; with `pt ~ 0.968`, 1000x lower loss.
+- Best setting found: `gamma = 2`, `alpha = 0.25`.
+
+### RetinaNet Architecture
+
+- **Backbone**: ResNet + Feature Pyramid Network (FPN), pyramid levels P3-P7 with C=256 channels.
+- **Classification subnet**: Small FCN with four 3x3 conv layers (256 filters, ReLU) + final 3x3 conv with KA filters + sigmoid. Shared across all pyramid levels.
+- **Box regression subnet**: Identical structure to classification subnet, terminates in 4A linear outputs per location. Class-agnostic.
+- **Anchors**: 9 anchors per level (3 aspect ratios x 3 scales), covering 32-813 pixel range.
+- **Initialization**: Prior probability pi=0.01 for foreground class at initialization to prevent instability from background class dominance.
+
+## Experimental Results
+
+### Ablation Studies (ResNet-50-FPN, COCO minival)
+
+| Loss | gamma | alpha | AP | AP50 | AP75 |
+|------|-------|-------|------|------|------|
+| CE (balanced) | 0 | 0.75 | 31.1 | 49.4 | 33.0 |
+| FL | 0.5 | 0.75 | 31.4 | 49.9 | 33.1 |
+| FL | 1.0 | 0.50 | 32.9 | 51.7 | 35.2 |
+| **FL** | **2.0** | **0.25** | **34.0** | **52.5** | **36.5** |
+| FL | 5.0 | 0.25 | 32.2 | 49.6 | 34.8 |
+
+### FL vs. OHEM (ResNet-101-FPN)
+
+| Method | AP | AP50 | AP75 |
+|--------|------|------|------|
+| OHEM (best) | 32.8 | 50.3 | 35.1 |
+| FL | **36.0** | **54.9** | **38.7** |
+
+FL outperforms the best OHEM variant by **3.2 AP points**.
+
+### State-of-the-Art Comparison (COCO test-dev)
+
+| Method | Backbone | AP | AP50 | AP75 |
+|--------|----------|------|------|------|
+| Faster R-CNN w FPN | ResNet-101-FPN | 36.2 | 59.1 | 39.0 |
+| Faster R-CNN by G-RMI | Inception-ResNet-v2 | 34.7 | 55.5 | 36.7 |
+| DSSD513 | ResNet-101-DSSD | 33.2 | 53.3 | 35.2 |
+| **RetinaNet** | **ResNet-101-FPN** | **39.1** | **59.1** | **42.3** |
+| **RetinaNet** | **ResNeXt-101-FPN** | **40.8** | **61.1** | **44.1** |
+
+### Speed vs. Accuracy
+
+- RetinaNet-101-600: 122ms inference, matching Faster R-CNN accuracy (36.0 AP) at similar speed.
+- RetinaNet-101-800: 198ms inference, 37.8 AP -- surpassing all prior methods.
+
+## Relevance to Turn-Taking / End-of-Turn Detection
+
+Focal loss is directly applicable to end-of-turn detection for BabelCast:
+
+1. **Class imbalance problem**: End-of-turn detection faces severe class imbalance -- the vast majority of audio frames are "continuing speech" (easy negatives), while actual turn-end boundaries are rare events. This is analogous to the foreground-background imbalance in object detection.
+
+2. **Down-weighting easy examples**: In a streaming ETD system, most 100ms frames are clearly mid-utterance. Focal loss would prevent these trivially classified frames from dominating the gradient, focusing learning on the ambiguous boundary cases (pauses vs. turn-ends).
+
+3. **Drop-in replacement**: Focal loss is a simple modification to standard cross entropy -- just adding `(1 - pt)^gamma` -- making it trivial to integrate into any binary or ternary classifier for end-of-turn detection (e.g., the GRU or Wav2Vec models in SpeculativeETD).
+
+4. **Hyperparameter robustness**: The paper shows `gamma = 2, alpha = 0.25` works well across settings, reducing the need for extensive tuning.
+
+5. **No sampling needed**: Unlike OHEM or hard negative mining, focal loss operates on all examples naturally, which is important for real-time streaming where we process every frame.

03-finetune-pipecat-pt/references/papers/focal_loss_lin_2017.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e6f45b97007009f32b82a3c19f35638d8da7022d8d4e4416b9078de91339f71
+size 1297358

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/01-kosmala-crible-2022-dual-status-filled-pauses.md

@@ -0,0 +1,74 @@
+---
+title: "The Dual Status of Filled Pauses: Evidence from Genre, Proficiency and Co-occurrence"
+authors:
+  - Loulou Kosmala
+  - Ludivine Crible
+year: 2022
+source_url: "https://doi.org/10.1177/00238309211010862"
+date_converted: 2026-03-16
+---
+
+## Abstract
+
+A corpus study examining the lexical vs. non-lexical status of filled pauses ("euh" and "eum") in spoken French. Analyzes their distribution across communication settings (prepared monologues vs. spontaneous conversations) and language proficiency levels (native vs. non-native French). Quantitative findings reveal differences in frequency, duration, position, and co-occurrence patterns. Qualitative analysis identifies two distinct patterns: (1) initial position clustered with a discourse marker (fluent, structuring use) vs. (2) medial position clustered with other hesitation markers (disfluent, processing use). The authors argue for a **dual status** of filled pauses based on formal, functional, and contextual features.
+
+## Key Findings Relevant to L2 Turn-Taking
+
+### French Filled Pause Forms
+- Two main variants in French: **"euh"** [schwa] and **"eum"** [nasal]
+- "Eum" is associated with longer delays and major discourse transitions/boundaries
+- "Euh" is the dominant form in both native and non-native French (84-92% of all FPs)
+
+### Genre Effects (DisReg Corpus -- French Native Speakers)
+- **Class presentations**: 6.8 FPs per 100 words (mean duration 415ms)
+- **Casual conversations**: 4.2 FPs per 100 words (mean duration 343ms)
+- More filled pauses in monologues than dialogues (significant, LL = 47.02, p < .001)
+- More "eum" in presentations (23%) vs. conversations (8%)
+- More initial-position FPs in presentations (40%) vs. conversations (24%)
+- More final-position FPs in conversations (14%) vs. presentations (3%) -- reflects **turn-yielding** function
+
+### Native vs. Non-Native French (SITAF Corpus)
+- **Native rate**: 4.4 FPs per 100 words (mean duration **378ms**)
+- **Non-native rate**: 5.3 FPs per 100 words (mean duration **524ms**)
+- Rate difference not significant, but **duration difference highly significant** (p < .001)
+- Both groups: ~60% medial position, ~30% initial, ~10% final
+- Non-native speakers had more **standalone and interrupted** positions (exclusive to learners)
+- Non-native speakers cluster FPs with other fluencemes more often (82% clustered vs. 72% for natives)
+- Longer fluenceme sequences in learner speech signal higher disruption
+
+### Co-occurrence with Discourse Markers
+- 69 instances of FP + discourse marker clusters found
+- Common French discourse markers paired with FPs: "donc" (so), "mais" (but), "ben" (well), "alors" (then/well), "en fait" (actually), "enfin" (I mean)
+- FP position shifts when clustered with discourse markers: **57% initial** (vs. 73% medial when isolated)
+- **Two distinct patterns emerge**:
+  1. **Fluent pattern**: FP + discourse marker at turn/phrase boundary (initial position) -- structuring function
+  2. **Disfluent pattern**: FP in medial position clustered with repetitions, lengthenings, silent pauses -- processing difficulty
+
+### Functional Analysis of Discourse Marker + FP Clusters
+Four discourse domains identified:
+- **Ideational**: connecting facts (e.g., "alors euh" = "then euh")
+- **Rhetorical**: expressing opinions (e.g., "mais euh" = "but euh")
+- **Sequential**: marking transitions (e.g., "donc euh" = "so euh")
+- **Interpersonal**: monitoring/agreement (e.g., "bah oui euh" = "well yes euh")
+
+## Specific Hesitation Pattern Data
+
+- Native French FP mean duration: **378ms** (SD=200ms)
+- Non-native French FP mean duration: **524ms** (SD=222ms) -- 146ms longer
+- FP duration is a more reliable index of proficiency than frequency
+- Duration of FPs with/without discourse markers: no significant difference (~433ms vs. ~467ms)
+- Example extreme disfluent sequence from a learner: 8 fluencemes in a row (lengthening + 3 silent pauses + 3 filled pauses + self-interruption), with pauses up to **1,177ms**
+
+## Implications for Turn-Taking Detection in L2 Speech
+
+1. **Dual function detection is essential**: The same sound "euh" can signal either (a) turn/phrase structuring (fluent, initial position + discourse marker) or (b) processing difficulty (disfluent, medial position + hesitation cluster). Both mean "don't take the turn yet" but for different reasons.
+
+2. **Duration as proficiency proxy**: Non-native FPs are ~150ms longer on average. A model trained on native French data will encounter systematically longer hesitations from L2 speakers. This duration difference should NOT be interpreted as turn-yielding.
+
+3. **Final-position FPs signal turn-yielding**: In conversations, 14% of FPs occur in final position (vs. 3% in monologues). The combination of FP + silent pause at utterance end is a reliable turn-yielding signal (e.g., "but he has a role euh (0.924)...").
+
+4. **Discourse marker + FP combinations**: Patterns like "donc euh" (so euh), "mais euh" (but euh) are strong turn-HOLD signals in French. A model should recognize these common French fluenceme clusters.
+
+5. **Cluster length matters**: Isolated FPs may go unnoticed; clustered FPs with repetitions and silent pauses signal genuine difficulty. Longer clusters = speaker is struggling but still holding the floor.
+
+6. **L2 speakers underuse discourse markers**: Non-native speakers rely more heavily on filled pauses alone, whereas native speakers combine FPs with rich discourse markers. The model should expect L2 French-Portuguese speakers to show heavy FP reliance with fewer Portuguese discourse markers.

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/01-kosmala-crible-2022-dual-status-filled-pauses.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46985a0e6f9345c0ae5a756f3bf13523e9ba2a76694c60ef2b861cf0aef67b4b
+size 631099

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/02-ekstedt-skantze-2022-prosody-turn-taking-vap.md

@@ -0,0 +1,68 @@
+---
+title: "How Much Does Prosody Help Turn-taking? Investigations using Voice Activity Projection Models"
+authors:
+  - Erik Ekstedt
+  - Gabriel Skantze
+year: 2022
+source_url: "https://doi.org/10.18653/v1/2023.acl-long.304"
+date_converted: 2026-03-16
+---
+
+## Abstract
+
+Investigates the role of prosody in turn-taking using the Voice Activity Projection (VAP) model, which incrementally models upcoming speech activity of interlocutors in a self-supervised manner without relying on explicit annotation of turn-taking events or explicit prosodic feature modeling. Through systematic manipulation of the speech signal (F0 flattening, intensity flattening, low-pass filtering, duration averaging, F0 shifting), the authors show that VAP models learn to utilize various prosodic aspects of speech for turn-taking prediction. The study uses both aggregate quantitative metrics on long-form conversations (Switchboard corpus) and controlled utterance-level experiments with synthesized short/long phrase pairs.
+
+## Key Findings Relevant to L2 Turn-Taking
+
+### VAP Model Architecture
+- Self-supervised model trained on raw waveforms (no hand-crafted features)
+- Predicts future voice activity for both speakers at every time frame
+- Tested at 20Hz, 50Hz, and 100Hz frame rates
+- Trained on **Switchboard corpus** (English telephone conversations, 260h)
+- Evaluated on 4 zero-shot tasks: shift vs. hold, shift prediction at voice activity, backchannel prediction, backchannel vs. turn-shift
+
+### Prosodic Perturbation Results
+Five signal manipulations tested:
+1. **Low-pass filter** (removes phonetic info, keeps F0 + intensity): Largest overall impact -- phonetic information is crucial
+2. **F0 flat** (removes pitch contour): Second most impactful -- intonation is important for disambiguating turn completion
+3. **Intensity flat** (removes loudness variation): Comparable impact to F0
+4. **F0 shift** (arbitrary pitch shift): Minimal effect on slower models (50Hz), larger effect on faster models (100Hz)
+5. **Duration average** (normalizes segment durations): Least important individual cue
+
+### Key Prosodic Finding: F0 Contour for Turn Projection
+- At **syntactically ambiguous completion points** (where lexical information is identical for both short and long utterances), the VAP model correctly uses prosody to distinguish turn-yielding from turn-holding
+- **Higher F0 rise + longer duration** at the last syllable = turn completion signal
+- The model predicts turn shifts **before** the utterance is actually complete -- it projects completions from prosodic dynamics
+- Even when F0 is flattened, the model still partially distinguishes hold/shift, indicating **redundant information in intensity and/or duration**
+
+### Relative Importance of Prosodic Cues
+1. **Phonetic information** (segmental, captured by spectral content): Most important overall
+2. **F0 contour** (intonation): Most important for disambiguating syntactically equivalent completion points
+3. **Intensity**: At least as important as pitch for general turn-taking
+4. **Duration**: Less important, but contributes as redundant cue
+
+### Model Frame Rate Findings
+- 20Hz and 50Hz models: comparable performance, more robust to perturbations
+- 100Hz models: more sensitive to phonetic information and acoustic artifacts
+- Lower frame rates preferred for computational efficiency and robustness
+
+## Specific Data Points
+
+- Human inter-turn gap: ~200ms average (Levinson & Torreira 2015)
+- Switchboard corpus: 2,400 dyadic telephone conversations, 260 hours
+- Short/long phrase pairs: 9 pairs, 10 TTS voices each (5 male, 5 female)
+- Three evaluation regions at short completion point: hold (start to -200ms), predictive (-200ms to end), reactive (last frame)
+
+## Implications for Turn-Taking Detection in L2 Speech
+
+1. **VAP models work without explicit prosodic features**: The self-supervised approach learns prosodic patterns from raw audio. This is crucial for L2 speech where prosodic patterns deviate from native norms -- the model can potentially learn L2-specific patterns if fine-tuned on L2 data.
+
+2. **F0 contour is the key disambiguation signal**: When words alone cannot determine turn boundaries (common in L2 speech with simpler syntax), the pitch contour becomes the primary cue. French speakers' intonation patterns in Portuguese will be a critical signal for the model.
+
+3. **Prosody is redundant and multi-dimensional**: Even removing one prosodic dimension (F0 or intensity) doesn't completely collapse turn-taking prediction. This redundancy is useful for L2 speech where some prosodic cues may be atypical.
+
+4. **50Hz frame rate is optimal**: For our Pipecat implementation, a 50Hz (20ms frame) VAP model provides the best balance of performance, robustness, and computational efficiency. This aligns with our current 16kHz/512-sample (32ms) Silero VAD window.
+
+5. **Pre-completion projection**: The VAP model predicts turn shifts BEFORE the speaker finishes, based on prosodic dynamics. This is essential for reducing response latency in real-time systems. For L2 speakers, this projection may be less reliable due to non-standard prosodic patterns, requiring L2-specific fine-tuning.
+
+6. **Cross-linguistic transfer needed**: The model was trained on English (Switchboard). For French speakers in Portuguese, it needs fine-tuning on Romance language conversation data. The underlying prosodic principles (F0 rise at completion, intensity patterns) likely transfer across Romance languages, but language-specific patterns must be learned.

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/02-ekstedt-skantze-2022-prosody-turn-taking-vap.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:472f85f79713f457e2ec5aa866a1e88cbd58d119a8b3e1561d50e7f1c1722661
+size 2825968

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/03-castillo-lopez-2025-survey-turn-taking-modeling.md

@@ -0,0 +1,77 @@
+---
+title: "A Survey of Recent Advances on Turn-taking Modeling in Spoken Dialogue Systems"
+authors:
+  - Galo Castillo-Lopez
+  - Gael de Chalendar
+  - Nasredine Semmar
+year: 2025
+source_url: "https://aclanthology.org/2025.iwsds-1.24/"
+date_converted: 2026-03-16
+---
+
+## Abstract
+
+A comprehensive review of recent methods on turn-taking modeling in spoken dialogue systems. Covers end-of-turn prediction, backchannel prediction, and multi-party turn-taking. Notes that 72% of reviewed works do not compare their methods with previous efforts, and argues that the lack of well-established benchmarks is a key challenge. Provides the first detailed review of datasets used in the field, discusses overlooked limitations, and examines new ideas and approaches since 2021. Published at IWSDS 2025.
+
+## Key Findings Relevant to L2 Turn-Taking
+
+### Turn-Taking Modeling Taxonomy
+Three main approaches to end-of-turn (EOU) prediction:
+1. **Silence-based**: Uses VAD + silence threshold (e.g., 700ms). Results in poor user experience due to unnaturalness. Current spoken dialogue systems wait 700-1000ms (vs. human 200ms average).
+2. **IPU-based (Inter-Pausal Unit)**: Predictions after each detected silence. Assumes turns cannot be taken while user speaks.
+3. **Continuous**: Predictions at every frame (e.g., every 50ms) regardless of silence. Most recent and promising approach.
+
+### Voice Activity Projection (VAP) Models -- Current State of the Art
+- VAP models predict future voice activity of both interlocutors incrementally
+- Self-supervised learning -- no explicit turn-taking event annotation needed
+- Best results in Japanese when trained on English + fine-tuned with Japanese data (vs. direct Japanese training)
+- Cross-lingual performance is poor without proper label alignment across datasets
+- Emerging trend: VAP models dominate continuous methods
+
+### Feature Importance for Turn-Taking
+- **Combined features outperform individual ones**: prosody + words > prosody alone or words alone
+- **Turn-taking cues have additive effect** in human communication
+- Word embeddings + acoustic features together improve backchannel detection
+- Gaze, head pose, and non-verbal features enhance predictions when available
+- **ASR-based linguistic features**: Fine-tuning wav2vec 2.0 for ASR outperforms acoustic-only features
+
+### Key Datasets
+| Dataset | Language | Duration | Dialogues |
+|---|---|---|---|
+| Switchboard | English | 260h | 2,400 |
+| Fisher Corpus | English | 1,960h | 11,700 |
+| NoXi Database | en/es/fr/de/it/ar/id | 25h | 84 |
+| AMI Meeting Corpus | English | 100h | 175 (multi-party) |
+
+- **Switchboard** is the dominant benchmark: used in 69% of backchannel and 41% of EOU papers
+- Most research is on **English and Japanese** -- very limited work on other languages
+- IPU silence thresholds vary from **50ms to 1s** across studies -- no standard
+
+### Backchannel Prediction
+- Backchannels are short feedback tokens ("mhm", "yeah") produced during another's speech
+- Key distinction: backchannel vs. actual interruption vs. non-lexical noise
+- Multi-task learning (sentiment + dialogue acts + backchannel) improves prediction
+- Context-aware models using BERT text embeddings + wav2vec acoustic embeddings show strong results
+
+### Open Challenges Identified
+1. **No standardized benchmarks**: Only 28% of papers compare with prior work
+2. **Multilinguality**: Very limited -- most work is English/Japanese only
+3. **Multi-party conversations**: Understudied, requires visual channel
+4. **Special populations**: Senior adults, mental health disorders need adapted models
+5. **LLMs are inefficient** at detecting mid-utterance turn opportunities
+
+## Implications for Turn-Taking Detection in L2 Speech
+
+1. **Continuous VAP models are the right approach**: For our fine-tuning task, continuous frame-level prediction (not silence-threshold-based) is the current state of the art. This aligns with our Pipecat architecture.
+
+2. **Cross-lingual fine-tuning works**: VAP models trained on English and fine-tuned on target language data outperform direct target-language training. This suggests training on English Switchboard + fine-tuning on French-Portuguese conversation data is a viable strategy.
+
+3. **Multi-feature approach is essential**: For L2 speakers, combining prosodic + lexical + timing features provides the best predictions. L2 speakers may have atypical prosody but more predictable syntax, or vice versa. The additive effect of features provides robustness.
+
+4. **Silence threshold tuning**: The survey notes thresholds from 50ms to 1s. For L2 speakers with longer pauses, the threshold must be adjusted upward. Our current 300ms SILENCE_HANGOVER_MS may need extension for L2 French speakers in Portuguese.
+
+5. **French is available in NoXi**: The NoXi database includes French conversations (among 7 languages) with 25h total. This could be a valuable fine-tuning resource for our French-specific model.
+
+6. **Backchannel handling**: L2 speakers may produce non-standard backchannels or transfer French backchannels ("ouais", "mmh") into Portuguese conversation. The model must distinguish these from turn-taking attempts.
+
+7. **Benchmarking gap**: Since 72% of papers don't compare methods, we should establish clear metrics for our L2 turn-taking task from the start, enabling future comparison.

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/03-castillo-lopez-2025-survey-turn-taking-modeling.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3fd4a6e8a77a46ad20634ba4b05c5731dbf4c692eb1acdb2fc1bb71062f12ee
+size 473840

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/04-cenoz-2000-pauses-hesitation-L2-production.md

@@ -0,0 +1,76 @@
+---
+title: "Pauses and Communication Strategies in Second Language Speech"
+authors:
+  - Jasone Cenoz
+year: 2000
+source_url: "https://eric.ed.gov/?id=ED426630"
+date_converted: 2026-03-16
+---
+
+## Abstract
+
+A study of silent and filled pauses in second language speech analyzing (1) which types of pauses are produced, (2) the functions of non-juncture pauses, (3) whether pauses co-occur with other hesitation phenomena, and (4) whether the occurrence of pauses is associated with second language proficiency. Subjects were 15 intermediate and advanced learners of English as a second language (L1 Spanish, university students). Each told a story in English which was recorded and transcribed. Silent and filled pauses at non-grammatical junctures were identified and analyzed.
+
+## Key Findings Relevant to L2 Turn-Taking
+
+### Pause Distribution
+- Total non-juncture pauses: **1,085** across 15 subjects
+- **64% silent pauses**, **36% filled pauses**
+- The most common filler was **"eh"** (transferred from L1 Spanish), demonstrating L1 transfer of hesitation markers
+- Wide individual variation: filled pauses ranged from 4% to 74.5% of all pauses per individual
+
+### Silent Pause Durations
+| Duration Range | % of Pauses | Subjects Affected | Individual Variation |
+|---|---|---|---|
+| 200-1000ms | 70% | 100% | 39%-96% |
+| 1001-2000ms | 21% | 100% | 4%-39% |
+| 2001-4000ms | 7% | 40% | 10%-18% |
+| 4001ms+ | 2% | 40% | 1.5%-7% |
+
+### Functional Categories of Pauses
+| Function | Silent Pauses | Filled Pauses |
+|---|---|---|
+| Lexical (retrieval) | 36% | 26% |
+| Morphological | 5% | 1% |
+| Planning | 59% | 73% |
+
+- Both silent and filled pauses serve the same functions, but filled pauses are overwhelmingly used for general **planning** (73%)
+- More silent pauses associated with **lexical retrieval** than filled pauses
+
+### Co-occurrence with Other Hesitation Phenomena
+| Strategy | Silent Pauses | Filled Pauses |
+|---|---|---|
+| Pauses + other hesitations | 54% | 23% |
+| Only pauses | 46% | 77% |
+
+- Most common hesitation phenomena: **repetition, self-correction, and reformulation**
+- Silent pauses much more likely to co-occur with other repair strategies (54%) vs. filled pauses (23%)
+- Filled pauses function as **standalone repair devices**; silent pauses tend to **precede** other repairs
+
+### Proficiency Effects
+- Higher-proficiency learners produced **more total pauses** and **more filled pauses** (53% silent + 46% filled)
+- Lower-proficiency learners: 69% silent + 31% filled
+- Lower-proficiency learners used **more hesitation strategies combined with pauses** (62% for silent pauses) vs. higher-proficiency (38%)
+- Interpretation: high-proficiency learners need only time to retrieve information; low-proficiency learners need to vocalize different options
+
+## Specific Hesitation Pattern Data
+
+- Silent pause minimum threshold: **200ms**
+- Pause range: **205ms to 11,569ms**
+- Most pauses (70%) are under 1 second
+- Long pauses (>2s) only in 40% of subjects -- marks a subset of struggling speakers
+- L1 Spanish filler "eh" dominates filled pauses (L1 transfer effect)
+
+## Implications for Turn-Taking Detection in L2 Speech
+
+1. **Pause duration is critical**: 70% of L2 pauses are 200-1000ms, overlapping with natural turn-transition gaps (~200ms). A turn-taking model must distinguish these within-turn hesitation pauses from actual turn boundaries.
+
+2. **Filled pauses signal continuation**: Since filled pauses are primarily standalone floor-holding devices (77% occur without other hesitations), detecting "eh/euh/um" should strongly indicate the speaker intends to continue, NOT yield the turn.
+
+3. **L1 transfer of filler forms**: French speakers speaking Portuguese will likely transfer French "euh" rather than adopting Portuguese fillers. The model should recognize L1-influenced fillers as valid hold signals.
+
+4. **Proficiency paradox**: More proficient L2 speakers use MORE filled pauses, not fewer. The model should not interpret high filler rates as low fluency or turn-yielding intent.
+
+5. **Cluster detection**: When silent pauses co-occur with repetitions, self-corrections, or reformulations (54% of cases), this strongly signals ongoing speech processing, not turn completion. Detecting hesitation clusters is key to avoiding premature turn-taking.
+
+6. **Individual variation is massive**: Some speakers use almost no filled pauses (4%) while others fill 74.5% of their pauses. The model needs speaker adaptation or robust handling of diverse hesitation profiles.

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/04-cenoz-2000-pauses-hesitation-L2-production.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e6e1ea3fb31493034539cd59a0274c8bcc11e08ff4104b2fb882c5766106e7a
+size 238029

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/05-kosmala-2023-filled-pauses-pragmatic-markers-gaze-gesture.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b4003c9f58c976ea687d117ea25b507885d0472d440eaad0e8386880e036734
+size 920978

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/06-wu-2023-disfluencies-french-prosodic-params-filled-pauses.md

@@ -0,0 +1,76 @@
+---
+title: "Disfluencies in Continuous Speech in French: Prosodic Parameters of Filled Pauses and Vowel Lengthening"
+authors:
+  - Yaru Wu
+  - Ivana Didirkova
+  - Anne-Catherine Simon
+year: 2023
+source_url: null
+date_converted: 2026-03-16
+---
+
+## Abstract
+
+This study examines prosodic parameters in two types of disfluencies -- vowel lengthenings and filled pauses -- in approximately 2.5 hours of continuous French speech across 11 different speech genres (prepared, semi-prepared, and unprepared). Analyzes mean fundamental frequency (F0), pitch resets, and duration to compare disfluent syllables with surrounding fluent syllables as a function of speech preparation level. Results show that F0 is lower in filled pauses and disfluent vowel lengthenings than in fluent speech, with filled pauses produced at lower F0 than vowel lengthening. Larger pitch resets occur between disfluent units and their preceding contexts.
+
+## Key Findings Relevant to L2 Turn-Taking
+
+### Prosodic Characteristics of French Filled Pauses
+
+#### Fundamental Frequency (F0) Drop
+- **Fluent speech**: mean 19.01 ST (SD=5.97)
+- **Vowel lengthening**: mean 18.10 ST (SD=6.43) -- ~1 ST drop
+- **Filled pauses**: mean 17.90 ST (SD=7.31) -- ~1.1 ST drop
+- F0 drop in filled pauses vs. fluent speech:
+  - **Females**: 2.12 ST decrease (from 23.66 to 21.54 ST)
+  - **Males**: 1.75 ST decrease (from 16.72 to 14.97 ST)
+- Both disfluency types significantly lower than fluent speech (p < 0.001)
+
+#### F0 and Speech Preparation Level
+- **Unprepared speech**: FP average F0 = 18.89 ST
+- **Semi-prepared speech**: FP average F0 = 17.41 ST
+- **Prepared speech**: FP average F0 = 14.24 ST
+- Massive **4.66 ST drop** between unprepared and prepared speech for FPs
+- Vowel lengthening does NOT follow this trend (only 0.07 ST difference)
+- Fluent speech shows moderate 1.2 ST drop between unprepared and prepared
+
+#### Pitch Reset (Melodic Discontinuity)
+- Filled pauses create a **larger negative pitch reset** from the preceding syllable than vowel lengthening
+- Both FPs and lengthening produce significantly more negative pitch changes than fluent speech (p < 0.001)
+- No significant pitch reset difference between the disfluent unit and the FOLLOWING syllable
+- Key insight: the pitch drop **before** a filled pause is a reliable acoustic cue
+
+#### Duration
+- **Vowel lengthening** is significantly LONGER than **filled pauses** (p < 0.001)
+- Vowel lengthening mean: ~347ms; Filled pause mean: ~268ms (from prior French study by Grosman 2018)
+- Filled pauses are shorter in prepared speech and longer in unprepared speech
+- Vowel lengthening shows the opposite pattern: shorter in unprepared, longer in prepared/semi-prepared
+
+### Corpus Details (LOCAS-F)
+- Multi-genre French oral corpus (14 speech genres)
+- Analyzed: 2h38m of recordings
+- Data: >1,500 vowel prolongation sequences, >1,000 filled pauses, ~59,000 fluent syllables
+- Disfluency affects ~4% of all data (typical for non-pathological speech)
+- Inter-annotator agreement: kappa 0.86 for FPs (near perfect), 0.64 for lengthening (substantial)
+
+## Specific Hesitation Pattern Data
+
+- French filled pauses typically transcribed as **"euh"**
+- Duration range for hesitation vowels (cross-linguistic): **200ms to 650ms** (from Vasilescu et al. 2004, 8 languages)
+- French FP F0 value is similar to the **onset value of breath groups** for a given speaker
+- Mean FP duration in French: **268.4ms** (Grosman 2018)
+- Mean vowel lengthening duration in French: **347.25ms** (Grosman 2018)
+
+## Implications for Turn-Taking Detection in L2 Speech
+
+1. **F0 drop is a reliable filled pause detector**: Filled pauses in French show a consistent 1-2 ST drop in F0 compared to fluent speech. A turn-taking model can use this prosodic signature to identify FPs even without lexical recognition. This is especially useful when ASR may not reliably transcribe L2 "euh" sounds.
+
+2. **Pitch reset before FPs marks processing onset**: The significant negative pitch reset between the preceding syllable and the filled pause signals the START of a hesitation event. This could serve as an early warning that the speaker is about to hesitate but intends to continue.
+
+3. **Preparation level affects FP prosody**: In more spontaneous speech (like real-time conversation), FPs have HIGHER F0 (closer to fluent speech). In prepared speech, FPs drop dramatically in F0. Since L2 speakers in conversation are in "unprepared" mode, their FPs may be harder to distinguish from fluent speech by F0 alone.
+
+4. **Vowel lengthening is a separate hesitation signal**: French speakers (both L1 and L2) elongate vowels at word boundaries as a hesitation strategy. These are LONGER than filled pauses (~347ms vs. ~268ms) but show a smaller F0 drop. The model should recognize elongated schwa-like sounds as hesitation, not turn completion.
+
+5. **4% disfluency rate in native speech**: This baseline helps calibrate expectations. L2 speakers will show significantly higher rates, potentially 8-15% or more, meaning the model will encounter disfluency markers very frequently in L2 French-to-Portuguese speech.
+
+6. **Cross-linguistic prosodic transfer**: French speakers speaking Portuguese will likely transfer their French FP prosodic patterns (F0 drop magnitude, pitch reset patterns). The model should accommodate French-influenced prosodic contours on Portuguese hesitations.

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/06-wu-2023-disfluencies-french-prosodic-params-filled-pauses.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c77a54e77105dc6ad1f3306d0a991c9730f70ed327559f2686bc5897c8f6540c
+size 436423

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/07-ekstedt-2023-turn-taking-cues-speech-synthesis.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9422e0c9df73fd3e2a91bb7bea6eba9393ba5865ea91b36463e1d50c23c9215b
+size 294389

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/08-inoue-2024-realtime-turn-taking-vap.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:129a900833fb9a775f6abc8869de60f3526a3a489ed6fda5f8b267136a5a8cec
+size 496267

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/09-inoue-2024-multilingual-turn-taking-vap.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a12189557e7022189609c710cd8071f359ae4e05a3c755c7e1aceef1b19a86bc
+size 974569

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/10-hutin-2024-filled-pauses-conversational-convergence.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e48ef19c610ebb2f2dca05458c801d11f2899563ff165aa7b1a90c17a563501a
+size 951784

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/11-beatty-martinez-2020-codeswitching-bilingual-toolkit.md

@@ -0,0 +1,67 @@
+---
+title: "Codeswitching: A Bilingual Toolkit for Opportunistic Speech Planning"
+authors:
+  - Anne L. Beatty-Martinez
+  - Christian A. Navarro-Torres
+  - Paola E. Dussias
+year: 2020
+source_url: "https://doi.org/10.3389/fpsyg.2020.01699"
+date_converted: 2026-03-16
+---
+
+## Abstract
+
+Reviews codeswitching as a bilingual strategy for opportunistic speech planning. Recent discoveries show that codeswitching is not haphazard but subject to unique linguistic and cognitive constraints, and that bilinguals who codeswitch exhibit usage patterns conforming to community-based norms. The paper provides corpus evidence (from the Puerto Rico Codeswitching Map Task corpus of Spanish-English bilinguals) that codeswitching serves as a tool to navigate linguistic interference during production, enabling speakers to circumvent speech planning difficulties by opportunistically drawing from whichever language is most active or accessible.
+
+## Key Findings Relevant to L2 Turn-Taking
+
+### Codeswitching as a Fluency Strategy
+- Codeswitching is NOT random -- it is **structured and strategic**
+- Functions as a tool for **opportunistic speech planning**: bilinguals take advantage of whichever language's words/structures are most active to achieve communicative goals
+- Reduces the cost in time and resources during speech production
+- **93% of codeswitches occur at Intonation Unit (IU) boundaries** (Plaistowe 2015, from NMSEB corpus)
+- This means switches overwhelmingly happen at natural prosodic break points, not mid-phrase
+
+### Prosodic Signatures of Codeswitching
+- Speech rate changes before codeswitches: momentary **reorganization of prosodic and phonetic systems**
+- These changes serve a dual purpose:
+  1. Help the speaker negotiate lexical competition and minimize cross-language interference
+  2. Provide reliable **acoustic cues for listeners** to anticipate and process the switch
+- Codeswitching affects bilingual speech at different levels: word-level (speech rate of individual words) and sentence-level (speech rate within prosodic sentences)
+
+### Language Control Modes
+- **Single-language context**: Language control is COMPETITIVE -- one language suppressed at expense of other
+- **Codeswitching context**: Language control is COOPERATIVE -- coactivation maintained, items from both languages available for selection
+- Dense codeswitchers minimize language membership tagging and keep both languages active
+
+### Corpus Data (Puerto Rico Codeswitching Map Task)
+- 10 Spanish-English bilinguals (6 female), all native Spanish speakers
+- Equal self-reported proficiency in both languages (9.6/10 each)
+- ~2.5 hours of unscripted, task-oriented dialogs
+- Participants exposed to both languages: more Spanish with family, more English in media, equal among friends
+- Paired with in-group confederate (close friend from same speech community) -- this increased codeswitching 4x vs. out-group pairing
+
+### Variable Equivalence and Switch Sites
+- Bilinguals do NOT consistently avoid "conflict sites" between languages
+- Instead, they **opportunistically use** sites of variable equivalence (partial structural overlap between languages)
+- This challenges the strict "equivalence constraint" (Poplack 1980)
+
+## Specific Data Points
+
+- Self-reported proficiency: Spanish 9.6/10 (SD=0.8), English 9.6/10 (SD=0.5)
+- Mean age: 23.3 years (SD=1.8)
+- Codeswitching at IU boundaries: 93% (from related corpus study)
+
+## Implications for Turn-Taking Detection in L2 Speech
+
+1. **Codeswitches happen at prosodic boundaries**: Since 93% of codeswitches align with intonation unit boundaries, a turn-taking model should expect language switches at natural break points -- the same locations where turn transitions might occur. The model must not interpret a language switch as a turn-ending signal.
+
+2. **Prosodic reorganization before switches**: The speech rate and prosodic changes before a codeswitch could be confused with turn-ending prosody. For French speakers occasionally inserting French words while speaking Portuguese, the model needs to tolerate these prosodic perturbations without triggering a false turn-shift prediction.
+
+3. **L1-L2 fluency trade-off**: When French speakers struggle with Portuguese lexical retrieval, they may briefly switch to French (a natural bilingual strategy). The turn-taking model should recognize this as a hold signal (the speaker is using an alternative strategy to continue) rather than a breakdown signal.
+
+4. **Cooperative language mode in bilingual contexts**: If the conversation partner also speaks French, the French-Portuguese speaker may engage in cooperative codeswitching. The model should handle mixed-language utterances without treating language boundaries as turn boundaries.
+
+5. **Community norms matter**: Codeswitching frequency and patterns depend heavily on the interactional context and community norms. The model should be configurable for different bilingual settings (e.g., high-codeswitch vs. monolingual-target contexts).
+
+6. **Intonation unit alignment**: Since codeswitches cluster at IU boundaries, and IU boundaries are also key sites for turn-taking decisions, the model must weigh additional cues (gaze, content completeness, prosodic finality) at these ambiguous boundary points where both a codeswitch-and-continue and a turn-yield are possible.

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/11-beatty-martinez-2020-codeswitching-bilingual-toolkit.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40ac9947f5813c936324bc2e266a1a574f1edb9d72a0d18f851bc3ddadf1d25b
+size 1159649

03-finetune-pipecat-pt/references/papers/hesitation-l2-french/12-lingref-code-switching-bilinguals.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca2aa2330e88de35a7f0254aefd3c024d14ce282f3f73ee28914f656f62f7820
+size 874378