Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,31 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/english_fleurs_1511.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/english_fleurs_1512.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/english_fleurs_1545.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/english_fleurs_1546.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/english_fleurs_1578.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/english_fleurs_1595.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/english_fleurs_1637.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/english_fleurs_1639.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/english_fleurs_1645.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hindi_fleurs_1524.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hindi_fleurs_1526.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hindi_fleurs_1540.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hindi_fleurs_1549.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hindi_fleurs_1560.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hindi_fleurs_1581.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hindi_fleurs_1609.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hindi_fleurs_1620.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hindi_fleurs_1641.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hindi_fleurs_1645.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hinglish_hiacc_AD09072.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hinglish_hiacc_AD22092.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hinglish_hiacc_AD23025.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hinglish_hiacc_AD36007.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hinglish_hiacc_AD36064.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hinglish_hiacc_AD40049.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hinglish_hiacc_AD40103.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hinglish_hiacc_AD40158.wav filter=lfs diff=lfs merge=lfs -text
+database/audio/sample_clips/hinglish_hiacc_AD60096.wav filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,13 +1,32 @@
 ---
-title: Turn Detection
-emoji: 🌖
-colorFrom: yellow
-colorTo: blue
+title: Turn Detection Live Dashboard
+emoji: 🎙️
+colorFrom: blue
+colorTo: yellow
 sdk: gradio
-sdk_version: 6.26.0
-python_version: '3.12'
+sdk_version: 6.25.0
 app_file: app.py
 pinned: false
+license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Turn Detection — Live Dashboard
+
+A real-time dashboard for audio turn detection: deciding whether a speaker is
+finished talking or just pausing. Speak into the mic or replay a bundled clip
+and watch several models score the decision live, plotted against the
+waveform.
+
+## Models
+Smart Turn v3.2 (zero-shot), Whisper with trained heads, a Qwen-based semantic
+judge, the LiveKit end-of-turn detector, acoustic + semantic fusion, and a
+streaming semantic branch.
+
+## How to use
+1. Select one or more models.
+2. Record from the mic, or replay a sample clip / upload a recording.
+3. Each model's "turn complete" probability is drawn against the waveform; the
+   dashed line is the decision threshold.
+
+Trained head checkpoints and demo clips are bundled. Other model weights
+download on first use.

app.py

@@ -0,0 +1,23 @@
+import traceback
+import numpy as np
+from backend.orchestrator import run_pipeline
+from backend.presets import list_presets
+from frontend.layout import build_app
+
+def _warmup() -> None:
+    sr = 16000
+    dummy = np.zeros(sr, dtype=np.float32)
+    for preset in list_presets():
+        if not preset.get('available'):
+            continue
+        try:
+            run_pipeline(preset['config'], dummy, sample_rate=sr, preset_label=preset['label'])
+            print(f"[warmup] ready: {preset['label']}")
+        except Exception as exc:
+            print(f"[warmup] skipped {preset['label']}: {exc}")
+            traceback.print_exc()
+    print('[warmup] done')
+if __name__ == '__main__':
+    _warmup()
+    demo = build_app()
+    demo.launch(share=False)

backend/audio_utils.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+import numpy as np
+import config
+
+def truncate_or_left_pad(audio: np.ndarray, n_seconds: int=config.SMART_TURN_WINDOW_SECONDS, sample_rate: int=config.SAMPLE_RATE) -> np.ndarray:
+    audio = np.asarray(audio, dtype=np.float32)
+    max_samples = n_seconds * sample_rate
+    if len(audio) > max_samples:
+        return audio[-max_samples:]
+    if len(audio) < max_samples:
+        padding = max_samples - len(audio)
+        return np.pad(audio, (padding, 0), mode='constant', constant_values=0)
+    return audio

backend/capabilities.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+import json
+from dataclasses import dataclass
+from typing import Optional
+import config
+STATIC_CAPABILITIES: dict[str, dict] = {'gate.webrtcvad': {'available': True, 'reason': None, 'provenance': 'real_checkpoint'}, 'gate.silero_vad': {'available': True, 'reason': None, 'provenance': 'real_checkpoint'}, 'gate.none': {'available': True, 'reason': None, 'provenance': 'rule'}, 'acoustic.smart_turn_onnx': {'available': True, 'reason': None, 'provenance': 'real_checkpoint'}, 'semantic.qwen_local': {'available': True, 'reason': None, 'provenance': 'real_checkpoint'}, 'semantic.livekit_eou': {'available': True, 'reason': None, 'provenance': 'real_checkpoint'}, 'semantic.qwen_local_streaming': {'available': True, 'reason': None, 'provenance': 'architecture_reimplemented'}, 'semantic.groq_api': {'available': False, 'reason': 'no API key configured - set GROQ_API_KEY to enable (see docs/decision-log.md #17)', 'provenance': 'unavailable'}, 'semantic.openrouter_api': {'available': False, 'reason': 'no API key configured - set OPENROUTER_API_KEY to enable (see docs/decision-log.md #17)', 'provenance': 'unavailable'}, 'fusion.weighted_vote': {'available': True, 'reason': None, 'provenance': 'rule'}, 'fusion.easy_turn': {'available': False, 'reason': "Easy Turn's linguistic branch needs its own ASR component we haven't wired yet - checkpoint downloaded for reference only (see docs/decision-log.md #19)", 'provenance': 'unavailable'}, 'mode.full_duplex_bypass.moshi': {'available': False, 'reason': 'needs GPU VRAM not available on this machine; API routing deferred (see docs/decision-log.md #6, #17)', 'provenance': 'unavailable'}, 'mode.full_duplex_bypass.human1': {'available': False, 'reason': 'needs GPU VRAM not available on this machine; API routing deferred (see docs/decision-log.md #6, #17)', 'provenance': 'unavailable'}}
+
+@dataclass
+class CapabilityInfo:
+    key: str
+    available: bool
+    reason: Optional[str]
+    provenance: Optional[str]
+
+def _head_checkpoint_path(encoder: str, pooling: str, head: str) -> tuple:
+    stem = f'{encoder}_{pooling}_{head}'
+    ckpt = config.CHECKPOINTS_DIR / f'{stem}.pt'
+    meta = config.CHECKPOINTS_DIR / f'{stem}.metadata.json'
+    return (ckpt, meta)
+
+def head_capability_key(encoder: str, pooling: str, head: str) -> str:
+    return f'acoustic.head.{encoder}.{pooling}.{head}'
+
+def _check_trained_head(encoder: str, pooling: str, head: str) -> CapabilityInfo:
+    key = head_capability_key(encoder, pooling, head)
+    ckpt, meta = _head_checkpoint_path(encoder, pooling, head)
+    if not (ckpt.exists() and meta.exists()):
+        return CapabilityInfo(key=key, available=False, reason=f'not trained yet - run experiments/train_head.py to produce database/checkpoints/{encoder}_{pooling}_{head}.pt', provenance='unavailable')
+    try:
+        metadata = json.loads(meta.read_text())
+    except (OSError, json.JSONDecodeError) as exc:
+        return CapabilityInfo(key=key, available=False, reason=f'checkpoint metadata unreadable ({exc}) - retrain via experiments/train_head.py', provenance='unavailable')
+    if metadata.get('encoder') != encoder or metadata.get('pooling') != pooling or metadata.get('head') != head:
+        return CapabilityInfo(key=key, available=False, reason='checkpoint metadata does not match the requested (encoder, pooling, head) combination', provenance='unavailable')
+    return CapabilityInfo(key=key, available=True, reason=None, provenance='trained_by_us')
+
+def get(key: str) -> CapabilityInfo:
+    if key.startswith('acoustic.head.'):
+        _, _, encoder, pooling, head = key.split('.')
+        return _check_trained_head(encoder, pooling, head)
+    if key in STATIC_CAPABILITIES:
+        entry = STATIC_CAPABILITIES[key]
+        return CapabilityInfo(key=key, **entry)
+    raise KeyError(f'unknown capability key: {key!r}')
+
+def is_available(key: str) -> bool:
+    return get(key).available
+
+def list_trained_heads() -> list[CapabilityInfo]:
+    found = []
+    if not config.CHECKPOINTS_DIR.exists():
+        return found
+    for meta_path in sorted(config.CHECKPOINTS_DIR.glob('*.metadata.json')):
+        try:
+            metadata = json.loads(meta_path.read_text())
+        except (OSError, json.JSONDecodeError):
+            continue
+        encoder, pooling, head = (metadata.get('encoder'), metadata.get('pooling'), metadata.get('head'))
+        if not all([encoder, pooling, head]):
+            continue
+        info = _check_trained_head(encoder, pooling, head)
+        if info.available:
+            found.append(info)
+    return found

backend/easy_turn.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+from backend.capabilities import get
+from backend.types import StageResult
+
+def run(*_args, **_kwargs) -> StageResult:
+    info = get('fusion.easy_turn')
+    return StageResult(stage='fusion.easy_turn', timing_ms=0.0, output=None, available=False, reason=info.reason, provenance='unavailable')

backend/encoders.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+import math
+import time
+from typing import Literal
+import numpy as np
+import torch
+import config
+from backend.audio_utils import truncate_or_left_pad
+from backend.types import StageResult
+EncoderName = Literal['whisper_tiny', 'whisper_base', 'wav2vec2']
+ENCODER_WINDOW_SECONDS = {'whisper_tiny': 30, 'whisper_base': 30, 'wav2vec2': config.SMART_TURN_WINDOW_SECONDS}
+_ENCODER_OUTPUT_HOP_SAMPLES = 320
+_device = 'mps' if torch.backends.mps.is_available() else 'cpu'
+_whisper_models: dict[str, tuple] = {}
+_wav2vec2_model = None
+_wav2vec2_processor = None
+
+def _get_whisper(model_id: str):
+    if model_id not in _whisper_models:
+        from transformers import WhisperFeatureExtractor, WhisperModel
+        feature_extractor = WhisperFeatureExtractor(chunk_length=30)
+        model = WhisperModel.from_pretrained(model_id).to(_device).eval()
+        _whisper_models[model_id] = (feature_extractor, model)
+    return _whisper_models[model_id]
+
+def _get_wav2vec2():
+    global _wav2vec2_model, _wav2vec2_processor
+    if _wav2vec2_model is None:
+        from transformers import Wav2Vec2FeatureExtractor, Wav2Vec2Model
+        _wav2vec2_processor = Wav2Vec2FeatureExtractor.from_pretrained(config.WAV2VEC2_ID)
+        _wav2vec2_model = Wav2Vec2Model.from_pretrained(config.WAV2VEC2_ID).to(_device).eval()
+    return (_wav2vec2_processor, _wav2vec2_model)
+
+def _valid_output_frames(original_num_samples: int, window_seconds: int, sample_rate: int) -> int:
+    real_samples = min(original_num_samples, window_seconds * sample_rate)
+    return max(1, math.ceil(real_samples / _ENCODER_OUTPUT_HOP_SAMPLES))
+
+@torch.inference_mode()
+def _encode_whisper(model_id: str, audio: np.ndarray, sample_rate: int) -> tuple[np.ndarray, int]:
+    feature_extractor, model = _get_whisper(model_id)
+    window_seconds = ENCODER_WINDOW_SECONDS['whisper_tiny']
+    window_samples = window_seconds * sample_rate
+    valid_frames = _valid_output_frames(len(audio), window_seconds, sample_rate)
+    padded = truncate_or_left_pad(audio, window_seconds, sample_rate)
+    inputs = feature_extractor(padded, sampling_rate=sample_rate, return_tensors='pt', padding='max_length', max_length=window_samples, truncation=True, do_normalize=True)
+    input_features = inputs.input_features.to(_device)
+    encoder = model.get_encoder()
+    hidden_states = encoder(input_features).last_hidden_state
+    hidden_states = hidden_states.squeeze(0).float().cpu().numpy()
+    valid_frames = min(valid_frames, hidden_states.shape[0])
+    return (hidden_states, valid_frames)
+
+@torch.inference_mode()
+def _encode_wav2vec2(audio: np.ndarray, sample_rate: int) -> tuple[np.ndarray, int]:
+    processor, model = _get_wav2vec2()
+    window_seconds = ENCODER_WINDOW_SECONDS['wav2vec2']
+    valid_frames = _valid_output_frames(len(audio), window_seconds, sample_rate)
+    padded = truncate_or_left_pad(audio, window_seconds, sample_rate)
+    inputs = processor(padded, sampling_rate=sample_rate, return_tensors='pt')
+    input_values = inputs.input_values.to(_device)
+    hidden_states = model(input_values).last_hidden_state
+    hidden_states = hidden_states.squeeze(0).float().cpu().numpy()
+    valid_frames = min(valid_frames, hidden_states.shape[0])
+    return (hidden_states, valid_frames)
+
+def extract_hidden_states(encoder: EncoderName, audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> tuple[np.ndarray, int]:
+    if encoder == 'whisper_tiny':
+        return _encode_whisper(config.WHISPER_TINY_ID, audio, sample_rate)
+    if encoder == 'whisper_base':
+        return _encode_whisper(config.WHISPER_BASE_ID, audio, sample_rate)
+    if encoder == 'wav2vec2':
+        return _encode_wav2vec2(audio, sample_rate)
+    raise ValueError(f'unknown encoder: {encoder!r}')
+
+def run(encoder: EncoderName, audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> StageResult:
+    start = time.perf_counter()
+    hidden_states, valid_length = extract_hidden_states(encoder, audio, sample_rate)
+    timing_ms = (time.perf_counter() - start) * 1000
+    return StageResult(stage=f'encoder.{encoder}', timing_ms=timing_ms, output={'hidden_states': hidden_states, 'valid_length': valid_length}, available=True, provenance='real_checkpoint')

backend/fusion.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+import time
+from typing import Optional
+from backend.types import StageResult
+
+def weighted_vote(acoustic_probability: float, semantic_verdict: Optional[str]=None, semantic_probability: Optional[float]=None, acoustic_weight: float=0.6) -> dict:
+    if semantic_verdict == 'wait':
+        return {'decision': 'incomplete', 'probability': min(acoustic_probability, 0.2), 'rule': "semantic 'wait' verdict overrides acoustic signal"}
+    if semantic_probability is not None:
+        semantic_score = semantic_probability
+    else:
+        semantic_score = 1.0 if semantic_verdict == 'complete' else 0.0
+    fused_probability = acoustic_weight * acoustic_probability + (1 - acoustic_weight) * semantic_score
+    decision = 'complete' if fused_probability > 0.5 else 'incomplete'
+    return {'decision': decision, 'probability': fused_probability, 'rule': f'weighted_vote(acoustic_weight={acoustic_weight})'}
+
+def run(acoustic_probability: float, semantic_verdict: Optional[str]=None, semantic_probability: Optional[float]=None, acoustic_weight: float=0.6) -> StageResult:
+    start = time.perf_counter()
+    result = weighted_vote(acoustic_probability, semantic_verdict, semantic_probability, acoustic_weight)
+    timing_ms = (time.perf_counter() - start) * 1000
+    return StageResult(stage='fusion.weighted_vote', timing_ms=timing_ms, output=result, available=True, provenance='rule')

backend/gate.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+import time
+import numpy as np
+import webrtcvad
+import config
+from backend.types import StageResult
+_silero_model = None
+_silero_utils = None
+
+def _get_silero():
+    global _silero_model, _silero_utils
+    if _silero_model is None:
+        import torch
+        _silero_model, _silero_utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', model='silero_vad', trust_repo=True)
+    return (_silero_model, _silero_utils)
+
+def _float_audio(audio: np.ndarray) -> np.ndarray:
+    audio = np.asarray(audio)
+    if audio.dtype.kind == 'i':
+        return audio.astype(np.float32) / 32768.0
+    return audio.astype(np.float32)
+
+def _pcm16_bytes(audio_f32: np.ndarray) -> bytes:
+    clipped = np.clip(audio_f32, -1.0, 1.0)
+    return (clipped * 32767.0).astype(np.int16).tobytes()
+
+def run_webrtcvad(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE, aggressiveness: int=2) -> StageResult:
+    start = time.perf_counter()
+    audio_f32 = _float_audio(audio)
+    pcm = _pcm16_bytes(audio_f32)
+    frame_ms = 30
+    frame_bytes = int(sample_rate * (frame_ms / 1000.0)) * 2
+    vad = webrtcvad.Vad(aggressiveness)
+    speech_frames = 0
+    total_frames = 0
+    for offset in range(0, len(pcm) - frame_bytes + 1, frame_bytes):
+        frame = pcm[offset:offset + frame_bytes]
+        total_frames += 1
+        if vad.is_speech(frame, sample_rate):
+            speech_frames += 1
+    speech_detected = total_frames > 0 and speech_frames / total_frames > 0.1
+    timing_ms = (time.perf_counter() - start) * 1000
+    return StageResult(stage='gate.webrtcvad', timing_ms=timing_ms, output={'speech_detected': speech_detected, 'speech_frame_ratio': speech_frames / total_frames if total_frames else 0.0}, available=True, provenance='real_checkpoint')
+
+def run_silero(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE, threshold: float=0.5) -> StageResult:
+    import torch
+    start = time.perf_counter()
+    model, utils = _get_silero()
+    get_speech_timestamps = utils[0]
+    audio_f32 = _float_audio(audio)
+    tensor = torch.from_numpy(audio_f32)
+    timestamps = get_speech_timestamps(tensor, model, sampling_rate=sample_rate, threshold=threshold)
+    speech_detected = len(timestamps) > 0
+    timing_ms = (time.perf_counter() - start) * 1000
+    return StageResult(stage='gate.silero_vad', timing_ms=timing_ms, output={'speech_detected': speech_detected, 'speech_segments': timestamps}, available=True, provenance='real_checkpoint')
+
+def run_none(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> StageResult:
+    return StageResult(stage='gate.none', timing_ms=0.0, output={'speech_detected': True}, available=True, provenance='rule')
+
+def run(gate: str, audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE, **kwargs) -> StageResult:
+    if gate == 'webrtcvad':
+        return run_webrtcvad(audio, sample_rate, aggressiveness=kwargs.get('vad_aggressiveness', 2))
+    if gate == 'silero_vad':
+        return run_silero(audio, sample_rate)
+    if gate == 'none':
+        return run_none(audio, sample_rate)
+    raise ValueError(f'unknown gate: {gate!r}')

backend/heads.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+import json
+from pathlib import Path
+from typing import Optional
+import torch
+import torch.nn as nn
+import config
+
+class LinearHead(nn.Module):
+
+    def __init__(self, input_dim: int):
+        super().__init__()
+        self.linear = nn.Linear(input_dim, 1)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.linear(x).squeeze(-1)
+
+class MLPHead(nn.Module):
+
+    def __init__(self, input_dim: int, hidden_dim: int=128):
+        super().__init__()
+        self.net = nn.Sequential(nn.Linear(input_dim, hidden_dim), nn.ReLU(), nn.Linear(hidden_dim, 1))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.net(x).squeeze(-1)
+
+def build_head(head_type: str, input_dim: int) -> nn.Module:
+    if head_type == 'linear':
+        return LinearHead(input_dim)
+    if head_type == 'mlp':
+        return MLPHead(input_dim)
+    raise ValueError(f'unknown head_type: {head_type!r}')
+
+def checkpoint_paths(encoder: str, pooling: str, head: str) -> tuple[Path, Path]:
+    stem = f'{encoder}_{pooling}_{head}'
+    return (config.CHECKPOINTS_DIR / f'{stem}.pt', config.CHECKPOINTS_DIR / f'{stem}.metadata.json')
+
+def save(encoder: str, pooling: str, head_type: str, input_dim: int, head_state_dict: dict, metrics: dict, pooling_state_dict: Optional[dict]=None, trained_on: Optional[str]=None) -> None:
+    config.CHECKPOINTS_DIR.mkdir(parents=True, exist_ok=True)
+    ckpt_path, meta_path = checkpoint_paths(encoder, pooling, head_type)
+    torch.save({'head_state_dict': head_state_dict, 'pooling_state_dict': pooling_state_dict, 'input_dim': input_dim}, ckpt_path)
+    metadata = {'encoder': encoder, 'pooling': pooling, 'head': head_type, 'input_dim': input_dim, 'metrics': metrics}
+    if trained_on is not None:
+        metadata['trained_on'] = trained_on
+    meta_path.write_text(json.dumps(metadata, indent=2))
+
+def load(encoder: str, pooling: str, head_type: str) -> tuple[nn.Module, Optional[dict], dict]:
+    ckpt_path, meta_path = checkpoint_paths(encoder, pooling, head_type)
+    if not (ckpt_path.exists() and meta_path.exists()):
+        raise FileNotFoundError(f'no trained checkpoint at {ckpt_path} - run experiments/train_head.py first (see backend/capabilities.py for the exact command).')
+    metadata = json.loads(meta_path.read_text())
+    blob = torch.load(ckpt_path, map_location='cpu', weights_only=True)
+    head = build_head(head_type, blob['input_dim'])
+    head.load_state_dict(blob['head_state_dict'])
+    head.eval()
+    return (head, blob.get('pooling_state_dict'), metadata)

backend/orchestrator.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+import time
+from typing import Optional
+import numpy as np
+import torch
+import config as cfg
+from backend import encoders, fusion, gate, heads, pooling, semantic, smart_turn_onnx, streaming_semantic
+from backend.capabilities import get as get_capability
+from backend.capabilities import head_capability_key
+from backend.types import PipelineConfig, PipelineResult, StageResult, StageUnavailableError
+
+def _require(capability_key: str) -> None:
+    info = get_capability(capability_key)
+    if not info.available:
+        raise StageUnavailableError(f'{capability_key} is not available: {info.reason}')
+
+def _run_custom_acoustic_branch(pc: PipelineConfig, audio: np.ndarray, sample_rate: int) -> tuple[StageResult, StageResult]:
+    head_key = head_capability_key(pc.encoder, pc.pooling, pc.head)
+    _require(head_key)
+    encoder_result = encoders.run(pc.encoder, audio, sample_rate)
+    hidden_states = encoder_result.output['hidden_states']
+    valid_length = encoder_result.output['valid_length']
+    head_module, pooling_state_dict, metadata = heads.load(pc.encoder, pc.pooling, pc.head)
+    pool_start = time.perf_counter()
+    with torch.inference_mode():
+        if pc.pooling == 'mean':
+            pooled = pooling.mean_pool(hidden_states, valid_length)
+            pooled_tensor = torch.from_numpy(pooled).float().unsqueeze(0)
+        elif pc.pooling == 'cross_attention':
+            attn_pool = pooling.CrossAttentionPool(hidden_dim=hidden_states.shape[-1])
+            attn_pool.load_state_dict(pooling_state_dict)
+            attn_pool.eval()
+            hs_tensor = torch.from_numpy(hidden_states).float().unsqueeze(0)
+            vl_tensor = torch.tensor([valid_length])
+            pooled_tensor = attn_pool(hs_tensor, vl_tensor)
+        else:
+            raise ValueError(f'unknown pooling: {pc.pooling!r}')
+        logit = head_module(pooled_tensor)
+        probability = torch.sigmoid(logit).item()
+    pool_head_ms = (time.perf_counter() - pool_start) * 1000
+    decision = 'complete' if probability > 0.5 else 'incomplete'
+    head_result = StageResult(stage=f'acoustic.custom.{pc.encoder}.{pc.pooling}.{pc.head}', timing_ms=pool_head_ms, output={'decision': decision, 'probability': probability, 'metrics_at_train_time': metadata.get('metrics')}, available=True, provenance='trained_by_us')
+    return (encoder_result, head_result)
+
+def run_pipeline(pc: PipelineConfig, audio: np.ndarray, sample_rate: int=cfg.SAMPLE_RATE, preset_label: Optional[str]=None) -> PipelineResult:
+    if pc.full_duplex != 'off':
+        _require(f'mode.full_duplex_bypass.{pc.full_duplex}')
+        raise StageUnavailableError('full-duplex bypass is not implemented in this pass')
+    stage_results: list[StageResult] = []
+    start = time.perf_counter()
+    _require(f'gate.{pc.gate}')
+    gate_result = gate.run(pc.gate, audio, sample_rate, vad_aggressiveness=pc.vad_aggressiveness)
+    stage_results.append(gate_result)
+    acoustic_result = None
+    acoustic_probability = None
+    acoustic_provenance = None
+    if pc.encoder == 'none':
+        pass
+    elif pc.encoder == 'smart_turn_onnx':
+        _require('acoustic.smart_turn_onnx')
+        acoustic_result = smart_turn_onnx.run(audio, sample_rate)
+        stage_results.append(acoustic_result)
+        acoustic_probability = acoustic_result.output['probability']
+        acoustic_provenance = 'real_checkpoint'
+    else:
+        encoder_result, head_result = _run_custom_acoustic_branch(pc, audio, sample_rate)
+        stage_results.extend([encoder_result, head_result])
+        acoustic_result = head_result
+        acoustic_probability = head_result.output['probability']
+        acoustic_provenance = 'trained_by_us'
+    transcript = None
+    semantic_verdict = None
+    semantic_probability = None
+    if pc.semantic != 'off':
+        _require(f'semantic.{pc.semantic}')
+        if pc.semantic == 'qwen_local':
+            semantic_result = semantic.run(audio, sample_rate, temperature=pc.semantic_temperature)
+            stage_results.append(semantic_result)
+            transcript = semantic_result.output['transcript']
+            semantic_verdict = semantic_result.output['verdict']
+        elif pc.semantic == 'qwen_local_streaming':
+            semantic_result = streaming_semantic.run(audio, sample_rate, temperature=pc.semantic_temperature)
+            stage_results.append(semantic_result)
+            transcript = semantic_result.output['transcript']
+            semantic_verdict = semantic_result.output['verdict']
+        elif pc.semantic == 'livekit_eou':
+            semantic_result = semantic.run_livekit(audio, sample_rate)
+            stage_results.append(semantic_result)
+            transcript = semantic_result.output['transcript']
+            semantic_probability = semantic_result.output['probability']
+        else:
+            raise StageUnavailableError(f'semantic provider {pc.semantic!r} is not implemented yet')
+    if pc.fusion == 'weighted_vote':
+        _require('fusion.weighted_vote')
+        fusion_result = fusion.run(acoustic_probability, semantic_verdict, semantic_probability, pc.acoustic_weight)
+        stage_results.append(fusion_result)
+        decision = fusion_result.output['decision']
+        probability = fusion_result.output['probability']
+        result_provenance = 'rule'
+    elif pc.fusion == 'easy_turn':
+        _require('fusion.easy_turn')
+        raise StageUnavailableError('unreachable')
+    elif pc.encoder == 'none':
+        if semantic_probability is not None:
+            decision = 'complete' if semantic_probability > 0.5 else 'incomplete'
+            probability = semantic_probability
+            result_provenance = 'real_checkpoint'
+        else:
+            decision = 'complete' if semantic_verdict == 'complete' else 'incomplete'
+            probability = None
+            result_provenance = 'architecture_reimplemented' if pc.semantic == 'qwen_local_streaming' else 'real_checkpoint'
+    else:
+        decision = acoustic_result.output['decision']
+        probability = acoustic_probability
+        result_provenance = acoustic_provenance
+    total_latency_ms = (time.perf_counter() - start) * 1000
+    return PipelineResult(config=pc, decision=decision, probability=probability, transcript=transcript, semantic_verdict=semantic_verdict, stage_results=stage_results, total_latency_ms=total_latency_ms, provenance=result_provenance, preset_label=preset_label)

backend/pooling.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+import numpy as np
+import torch
+import torch.nn as nn
+
+def mean_pool(hidden_states: np.ndarray, valid_length: int) -> np.ndarray:
+    valid_length = max(1, min(valid_length, hidden_states.shape[0]))
+    return hidden_states[-valid_length:].mean(axis=0)
+
+class CrossAttentionPool(nn.Module):
+
+    def __init__(self, hidden_dim: int):
+        super().__init__()
+        self.hidden_dim = hidden_dim
+        self.query = nn.Parameter(torch.randn(hidden_dim) * hidden_dim ** (-0.5))
+
+    def forward(self, hidden_states: torch.Tensor, valid_lengths: torch.Tensor) -> torch.Tensor:
+        batch, seq_len, dim = hidden_states.shape
+        positions = torch.arange(seq_len, device=hidden_states.device).unsqueeze(0)
+        start_idx = (seq_len - valid_lengths).unsqueeze(1)
+        mask = positions >= start_idx
+        scores = hidden_states @ self.query / dim ** 0.5
+        scores = scores.masked_fill(~mask, float('-inf'))
+        weights = torch.softmax(scores, dim=-1)
+        pooled = torch.einsum('bt,btd->bd', weights, hidden_states)
+        return pooled

backend/presets.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from backend.capabilities import get as get_capability
+from backend.capabilities import head_capability_key
+from backend.types import PipelineConfig, Provenance
+
+@dataclass
+class Preset:
+    label: str
+    config: PipelineConfig
+    approximates: str
+    static_provenance: Provenance
+
+def _required_capability_keys(pc: PipelineConfig) -> list[str]:
+    keys = [f'gate.{pc.gate}']
+    if pc.encoder == 'smart_turn_onnx':
+        keys.append('acoustic.smart_turn_onnx')
+    elif pc.encoder != 'none':
+        keys.append(head_capability_key(pc.encoder, pc.pooling, pc.head))
+    if pc.semantic != 'off':
+        keys.append(f'semantic.{pc.semantic}')
+    if pc.fusion != 'off':
+        keys.append(f'fusion.{pc.fusion}')
+    return keys
+_PRESET_DEFS: list[Preset] = [Preset(label='Smart Turn v3.2 (zero-shot)', config=PipelineConfig(gate='silero_vad', encoder='smart_turn_onnx', semantic='off', fusion='off'), approximates='Smart Turn v3.2 (Pipecat)', static_provenance='real_checkpoint'), Preset(label='Whisper-Tiny + Mean-Pool + Linear (trained)', config=PipelineConfig(gate='silero_vad', encoder='whisper_tiny', pooling='mean', head='linear', semantic='off', fusion='off'), approximates="Smart Turn v2's recipe (mean pool + linear), retrained by us", static_provenance='trained_by_us'), Preset(label='Whisper-Tiny + Mean-Pool + MLP (trained)', config=PipelineConfig(gate='silero_vad', encoder='whisper_tiny', pooling='mean', head='mlp', semantic='off', fusion='off'), approximates='', static_provenance='trained_by_us'), Preset(label='Whisper-Tiny + Cross-Attn + Linear (trained)', config=PipelineConfig(gate='silero_vad', encoder='whisper_tiny', pooling='cross_attention', head='linear', semantic='off', fusion='off'), approximates="Smart Turn v3.2's recipe (cross-attn pool + linear), retrained by us on real Hindi/Hinglish audio (HiACC + FLEURS) after the same architecture scored worst OOD (1/5) trained on the challenge's synthetic TTS data - real data fixed it outright (5/5). See docs/decision-log.md #38, experiments/results/dataset_comparison.md.", static_provenance='trained_by_us'), Preset(label='TEN Turn Detection (prompted)', config=PipelineConfig(gate='silero_vad', encoder='none', semantic='qwen_local', fusion='off'), approximates='TEN Turn Detection - approximated with local Qwen2.5-0.5B instead of their 7B (docs/decision-log.md #11)', static_provenance='real_checkpoint'), Preset(label='LiveKit End-of-Turn Detector (zero-shot)', config=PipelineConfig(gate='silero_vad', encoder='none', semantic='livekit_eou', fusion='off'), approximates='livekit/turn-detector - a real checkpoint, not an approximation (docs/decision-log.md #33)', static_provenance='real_checkpoint'), Preset(label='Acoustic + Semantic Fusion (rule-based)', config=PipelineConfig(gate='silero_vad', encoder='whisper_tiny', pooling='mean', head='linear', semantic='livekit_eou', fusion='weighted_vote'), approximates="A zero-training combination of our trained acoustic head with LiveKit's real EOU probability (docs/decision-log.md #33) - NOT Easy Turn's actual jointly-trained fusion, and no 'wait' handling since LiveKit's model has no such class (traded away when this preset moved off qwen_local)", static_provenance='rule'), Preset(label='FastTurn Streaming (reimplemented)', config=PipelineConfig(gate='silero_vad', encoder='smart_turn_onnx', semantic='qwen_local_streaming', fusion='weighted_vote'), approximates="FastTurn's described streaming-fusion mechanism, reimplemented by us for Hindi/Hinglish with a CTC checkpoint (theainerd/Wav2Vec2-large-xlsr-hindi) - NOT their code or reported numbers, which were never released (docs/decision-log.md #23)", static_provenance='architecture_reimplemented'), Preset(label='Easy Turn (unavailable)', config=PipelineConfig(gate='silero_vad', encoder='whisper_tiny', pooling='mean', head='linear', semantic='qwen_local', fusion='easy_turn'), approximates="Easy Turn's real joint acoustic+linguistic fusion - NOT wired up this pass", static_provenance='unavailable')]
+
+def list_presets() -> list[dict]:
+    out = []
+    for preset in _PRESET_DEFS:
+        try:
+            infos = [get_capability(key) for key in _required_capability_keys(preset.config)]
+            failing = [i for i in infos if not i.available]
+            if failing:
+                available = False
+                reason = '; '.join((f'{i.key}: {i.reason}' for i in failing))
+                provenance = 'unavailable'
+            else:
+                available = True
+                reason = None
+                provenance = preset.static_provenance
+        except (KeyError, ValueError) as exc:
+            available = False
+            reason = f'preset is not runnable as configured: {exc}'
+            provenance = 'unavailable'
+        out.append({'label': preset.label, 'config': preset.config, 'approximates': preset.approximates, 'available': available, 'reason': reason, 'provenance': provenance})
+    return out

backend/semantic.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+import re
+import time
+import unicodedata
+import numpy as np
+import torch
+import config
+from backend.types import StageResult
+_device = 'mps' if torch.backends.mps.is_available() else 'cpu'
+_asr_pipeline = None
+_qwen_model = None
+_qwen_tokenizer = None
+_livekit_model = None
+_livekit_tokenizer = None
+_livekit_im_end_id = None
+PROMPT_TEMPLATE = 'You are analyzing a snippet of transcribed speech from a Hindi-English (Hinglish) conversation with a voice assistant. Code-switching between Hindi and English mid-sentence is normal and not a sign of incompleteness. Filler words like "matlab", "toh", "haan", "wo kya bolte hain", "um", "uh" indicate the speaker is still thinking and has NOT completed their turn.\n\nTranscript: "{transcript}"\n\nClassify whether the speaker\'s turn is:\n- complete: the utterance is a complete thought, the speaker is done\n- incomplete: the utterance is grammatically or semantically incomplete, more is coming\n- wait: the speaker is explicitly asking for a pause (e.g. "ek second", "hold on", "wait")\n\nRespond with exactly one word: complete, incomplete, or wait.'
+_VALID_LABELS = ('complete', 'incomplete', 'wait')
+
+def _get_asr_pipeline():
+    global _asr_pipeline
+    if _asr_pipeline is None:
+        from transformers import pipeline
+        _asr_pipeline = pipeline('automatic-speech-recognition', model=config.WHISPER_TINY_ID, device=_device if _device != 'mps' else -1)
+    return _asr_pipeline
+
+def _get_qwen():
+    global _qwen_model, _qwen_tokenizer
+    if _qwen_model is None:
+        from transformers import AutoModelForCausalLM, AutoTokenizer
+        _qwen_tokenizer = AutoTokenizer.from_pretrained(config.QWEN_LOCAL_ID)
+        _qwen_model = AutoModelForCausalLM.from_pretrained(config.QWEN_LOCAL_ID).to(_device).eval()
+    return (_qwen_model, _qwen_tokenizer)
+
+def _get_livekit():
+    global _livekit_model, _livekit_tokenizer, _livekit_im_end_id
+    if _livekit_model is None:
+        from transformers import AutoModelForCausalLM, AutoTokenizer
+        _livekit_tokenizer = AutoTokenizer.from_pretrained(config.LIVEKIT_TURN_DETECTOR_ID)
+        _livekit_model = AutoModelForCausalLM.from_pretrained(config.LIVEKIT_TURN_DETECTOR_ID).to(_device).eval()
+        _livekit_im_end_id = _livekit_tokenizer.convert_tokens_to_ids('<|im_end|>')
+    return (_livekit_model, _livekit_tokenizer, _livekit_im_end_id)
+_PUNCTUATION_RE = re.compile("[^\\w\\s'-]", re.UNICODE)
+_WHITESPACE_RE = re.compile('\\s+')
+
+def _normalize_for_livekit(transcript: str) -> str:
+    text = unicodedata.normalize('NFKC', transcript).lower()
+    text = _PUNCTUATION_RE.sub(' ', text)
+    return _WHITESPACE_RE.sub(' ', text).strip()
+
+def transcribe(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> str:
+    asr = _get_asr_pipeline()
+    result = asr({'raw': np.asarray(audio, dtype=np.float32), 'sampling_rate': sample_rate})
+    return result['text'].strip()
+
+def _parse_label(raw_text: str) -> str:
+    lowered = raw_text.lower()
+    for label in _VALID_LABELS:
+        if re.search(f'\\b{label}\\b', lowered):
+            return label
+    return 'incomplete'
+
+@torch.inference_mode()
+def classify_transcript(transcript: str, temperature: float=0.2) -> dict:
+    model, tokenizer = _get_qwen()
+    messages = [{'role': 'user', 'content': PROMPT_TEMPLATE.format(transcript=transcript)}]
+    prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+    inputs = tokenizer(prompt, return_tensors='pt').to(_device)
+    do_sample = temperature > 0
+    output_ids = model.generate(**inputs, max_new_tokens=8, do_sample=do_sample, temperature=temperature if do_sample else None, pad_token_id=tokenizer.eos_token_id)
+    generated = output_ids[0][inputs['input_ids'].shape[1]:]
+    raw_text = tokenizer.decode(generated, skip_special_tokens=True)
+    label = _parse_label(raw_text)
+    return {'verdict': label, 'raw_response': raw_text.strip()}
+
+def run(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE, temperature: float=0.2) -> StageResult:
+    start = time.perf_counter()
+    transcript = transcribe(audio, sample_rate)
+    result = classify_transcript(transcript, temperature)
+    timing_ms = (time.perf_counter() - start) * 1000
+    return StageResult(stage='semantic.qwen_local', timing_ms=timing_ms, output={'transcript': transcript, 'verdict': result['verdict'], 'raw_response': result['raw_response']}, available=True, provenance='real_checkpoint')
+
+@torch.inference_mode()
+def classify_transcript_livekit(transcript: str) -> float:
+    model, tokenizer, im_end_id = _get_livekit()
+    normalized = _normalize_for_livekit(transcript)
+    prompt = f'<|im_start|><|user|>{normalized}'
+    inputs = tokenizer(prompt, return_tensors='pt').to(_device)
+    logits = model(**inputs).logits[0, -1, :]
+    probs = torch.softmax(logits, dim=-1)
+    return probs[im_end_id].item()
+
+def run_livekit(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> StageResult:
+    start = time.perf_counter()
+    transcript = transcribe(audio, sample_rate)
+    probability = classify_transcript_livekit(transcript)
+    timing_ms = (time.perf_counter() - start) * 1000
+    return StageResult(stage='semantic.livekit_eou', timing_ms=timing_ms, output={'transcript': transcript, 'probability': probability}, available=True, provenance='real_checkpoint')

backend/smart_turn_onnx.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+import time
+from pathlib import Path
+import numpy as np
+import onnxruntime as ort
+from transformers import WhisperFeatureExtractor
+import config
+from backend.audio_utils import truncate_or_left_pad
+from backend.types import StageResult
+_session: ort.InferenceSession | None = None
+_feature_extractor: WhisperFeatureExtractor | None = None
+
+def _onnx_path() -> Path:
+    return config.MODELS_DIR / config.SMART_TURN_ONNX_FILENAME
+
+def _get_session() -> ort.InferenceSession:
+    global _session
+    if _session is None:
+        path = _onnx_path()
+        if not path.exists():
+            from huggingface_hub import hf_hub_download
+            config.MODELS_DIR.mkdir(parents=True, exist_ok=True)
+            hf_hub_download(repo_id=config.SMART_TURN_REPO, filename=config.SMART_TURN_ONNX_FILENAME, local_dir=config.MODELS_DIR)
+        _session = ort.InferenceSession(str(path))
+    return _session
+
+def _get_feature_extractor() -> WhisperFeatureExtractor:
+    global _feature_extractor
+    if _feature_extractor is None:
+        _feature_extractor = WhisperFeatureExtractor(chunk_length=config.SMART_TURN_WINDOW_SECONDS)
+    return _feature_extractor
+
+def preprocess(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> np.ndarray:
+    if sample_rate != config.SAMPLE_RATE:
+        raise ValueError(f'Smart Turn expects {config.SAMPLE_RATE}Hz mono PCM, got sample_rate={sample_rate}. Resample before calling this function.')
+    audio = truncate_or_left_pad(audio, config.SMART_TURN_WINDOW_SECONDS, sample_rate)
+    fe = _get_feature_extractor()
+    inputs = fe(audio, sampling_rate=sample_rate, return_tensors='np', padding='max_length', max_length=config.SMART_TURN_WINDOW_SAMPLES, truncation=True, do_normalize=True)
+    input_features = inputs.input_features.squeeze(0).astype(np.float32)
+    return np.expand_dims(input_features, axis=0)
+
+def run(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> StageResult:
+    start = time.perf_counter()
+    session = _get_session()
+    input_features = preprocess(audio, sample_rate)
+    outputs = session.run(None, {'input_features': input_features})
+    probability = float(outputs[0].flatten()[0])
+    prediction = 'complete' if probability > 0.5 else 'incomplete'
+    timing_ms = (time.perf_counter() - start) * 1000
+    return StageResult(stage='acoustic.smart_turn_onnx', timing_ms=timing_ms, output={'decision': prediction, 'probability': probability}, available=True, provenance='real_checkpoint')

backend/streaming_semantic.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+import math
+import time
+import numpy as np
+import torch
+import config
+from backend.semantic import classify_transcript
+from backend.types import StageResult
+_device = 'mps' if torch.backends.mps.is_available() else 'cpu'
+_ctc_model = None
+_ctc_processor = None
+DEFAULT_CHUNK_MS = 1000
+MAX_STEPS = 10
+
+def _get_ctc():
+    global _ctc_model, _ctc_processor
+    if _ctc_model is None:
+        from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
+        _ctc_processor = Wav2Vec2Processor.from_pretrained(config.HINDI_CTC_ID)
+        _ctc_model = Wav2Vec2ForCTC.from_pretrained(config.HINDI_CTC_ID).to(_device).eval()
+    return (_ctc_processor, _ctc_model)
+
+@torch.inference_mode()
+def transcribe_ctc_chunk(audio_prefix: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> str:
+    processor, model = _get_ctc()
+    inputs = processor(audio_prefix, sampling_rate=sample_rate, return_tensors='pt', padding=True)
+    input_values = inputs.input_values.to(_device)
+    logits = model(input_values).logits
+    predicted_ids = torch.argmax(logits, dim=-1)
+    text = processor.batch_decode(predicted_ids)[0]
+    return text.strip()
+
+def run(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE, chunk_ms: int=DEFAULT_CHUNK_MS, temperature: float=0.2) -> StageResult:
+    start = time.perf_counter()
+    audio = np.asarray(audio, dtype=np.float32)
+    total_audio_ms = len(audio) / sample_rate * 1000
+    chunk_samples = int(sample_rate * chunk_ms / 1000)
+    history: list[dict] = []
+    first_decisive_audio_ms = None
+    num_steps = min(MAX_STEPS, max(1, len(audio) // chunk_samples))
+    stride = max(chunk_samples, math.ceil(len(audio) / num_steps))
+    for step in range(1, num_steps + 1):
+        prefix_len = min(len(audio), step * stride)
+        prefix = audio[:prefix_len]
+        elapsed_audio_ms = prefix_len / sample_rate * 1000
+        step_start = time.perf_counter()
+        transcript_so_far = transcribe_ctc_chunk(prefix, sample_rate)
+        verdict = 'incomplete'
+        if transcript_so_far:
+            verdict = classify_transcript(transcript_so_far, temperature)['verdict']
+        step_ms = (time.perf_counter() - step_start) * 1000
+        history.append({'elapsed_audio_ms': round(elapsed_audio_ms, 1), 'transcript_so_far': transcript_so_far, 'verdict': verdict, 'step_processing_ms': round(step_ms, 1)})
+        if first_decisive_audio_ms is None and verdict != 'incomplete':
+            first_decisive_audio_ms = elapsed_audio_ms
+        if prefix_len >= len(audio):
+            break
+    final = history[-1]
+    timing_ms = (time.perf_counter() - start) * 1000
+    return StageResult(stage='semantic.qwen_local_streaming', timing_ms=timing_ms, output={'transcript': final['transcript_so_far'], 'verdict': final['verdict'], 'history': history, 'total_audio_ms': round(total_audio_ms, 1), 'first_decisive_audio_ms': first_decisive_audio_ms, 'eou_delay_saved_ms': round(total_audio_ms - first_decisive_audio_ms, 1) if first_decisive_audio_ms is not None else None}, available=True, provenance='architecture_reimplemented')

backend/types.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Any, Literal, Optional
+Provenance = Literal['real_checkpoint', 'trained_by_us', 'rule', 'architecture_reimplemented', 'unavailable']
+GateChoice = Literal['webrtcvad', 'silero_vad', 'none']
+EncoderChoice = Literal['whisper_tiny', 'whisper_base', 'wav2vec2', 'smart_turn_onnx', 'none']
+PoolingChoice = Literal['mean', 'cross_attention']
+HeadChoice = Literal['linear', 'mlp']
+SemanticChoice = Literal['off', 'qwen_local', 'qwen_local_streaming', 'livekit_eou', 'groq_api', 'openrouter_api']
+FusionChoice = Literal['off', 'weighted_vote', 'easy_turn']
+OutputClasses = Literal['binary', '3class', '4class']
+FullDuplexChoice = Literal['off', 'moshi', 'human1']
+
+class StageUnavailableError(RuntimeError):
+    pass
+
+@dataclass
+class StageResult:
+    stage: str
+    timing_ms: float
+    output: Any = None
+    available: bool = True
+    reason: Optional[str] = None
+    provenance: Optional[Provenance] = None
+
+@dataclass
+class PipelineConfig:
+    gate: GateChoice = 'silero_vad'
+    encoder: EncoderChoice = 'whisper_tiny'
+    pooling: PoolingChoice = 'mean'
+    head: HeadChoice = 'linear'
+    semantic: SemanticChoice = 'off'
+    fusion: FusionChoice = 'off'
+    output_classes: OutputClasses = 'binary'
+    full_duplex: FullDuplexChoice = 'off'
+    vad_aggressiveness: int = 2
+    silence_trigger_ms: int = 400
+    semantic_temperature: float = 0.2
+    acoustic_weight: float = 0.6
+
+    def __post_init__(self) -> None:
+        if self.encoder == 'none':
+            if self.semantic == 'off':
+                raise ValueError("encoder='none' requires the semantic branch to be on (nothing would decide).")
+            if self.fusion != 'off':
+                raise ValueError("encoder='none' has no acoustic score to fuse with - set fusion='off'.")
+        if self.encoder == 'smart_turn_onnx':
+            if self.pooling != 'mean' or self.head != 'linear':
+                raise ValueError("encoder='smart_turn_onnx' is a single opaque preset and cannot be combined with a separate pooling/head choice (see docs/decision-log.md #18).")
+        if self.fusion != 'off' and self.semantic == 'off':
+            raise ValueError('fusion requires the semantic branch to be on.')
+        if self.full_duplex != 'off':
+            pass
+
+@dataclass
+class PipelineResult:
+    config: PipelineConfig
+    decision: Optional[str] = None
+    probability: Optional[float] = None
+    transcript: Optional[str] = None
+    semantic_verdict: Optional[str] = None
+    stage_results: list[StageResult] = field(default_factory=list)
+    total_latency_ms: float = 0.0
+    provenance: Optional[Provenance] = None
+    preset_label: Optional[str] = None

config.py

@@ -0,0 +1,33 @@
+from pathlib import Path
+ROOT = Path(__file__).resolve().parent
+DATABASE_DIR = ROOT / 'database'
+AUDIO_DIR = DATABASE_DIR / 'audio'
+SAMPLE_CLIPS_DIR = AUDIO_DIR / 'sample_clips'
+EVAL_DIR = AUDIO_DIR / 'eval'
+OOD_TEST_CLIPS_DIR = EVAL_DIR / 'ood_test_clips'
+TRUNCATION_PILOT_DIR = EVAL_DIR / 'truncation_pilot'
+LIVE_RUNS_DIR = AUDIO_DIR / 'live_runs'
+CHECKPOINTS_DIR = DATABASE_DIR / 'checkpoints'
+MODELS_DIR = DATABASE_DIR / 'models'
+EASY_TURN_DIR = MODELS_DIR / 'easy_turn'
+CACHE_DIR = DATABASE_DIR / 'cache'
+DATA_CACHE_DIR = CACHE_DIR / 'dataset'
+OOD_CACHE_DIR = CACHE_DIR / 'ood_test'
+EXPERIMENTS_DIR = ROOT / 'experiments'
+EMBEDDINGS_CACHE_DIR = EXPERIMENTS_DIR / 'embeddings_cache'
+RESULTS_DIR = EXPERIMENTS_DIR / 'results'
+SAMPLE_RATE = 16000
+SMART_TURN_WINDOW_SECONDS = 8
+SMART_TURN_WINDOW_SAMPLES = SMART_TURN_WINDOW_SECONDS * SAMPLE_RATE
+WHISPER_TINY_ID = 'openai/whisper-tiny'
+WHISPER_BASE_ID = 'openai/whisper-base'
+WAV2VEC2_ID = 'facebook/wav2vec2-base'
+QWEN_LOCAL_ID = 'Qwen/Qwen2.5-0.5B-Instruct'
+HINDI_CTC_ID = 'theainerd/Wav2Vec2-large-xlsr-hindi'
+LIVEKIT_TURN_DETECTOR_ID = 'livekit/turn-detector'
+SMART_TURN_REPO = 'pipecat-ai/smart-turn-v3'
+SMART_TURN_ONNX_FILENAME = 'smart-turn-v3.2-cpu.onnx'
+DATASET_REPO = 'pipecat-ai/smart-turn-data-v3.2-train'
+DATASET_NUM_SHARDS_TOTAL = 83
+EASY_TURN_REPO = 'ASLP-lab/Easy-Turn'
+ENCODER_IDS = {'whisper_tiny': WHISPER_TINY_ID, 'whisper_base': WHISPER_BASE_ID, 'wav2vec2': WAV2VEC2_ID}

database/audio/sample_clips/english_fleurs_1511.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9dde919c8a85ece3116cf42fa3f2d052a9174319a4062c975631c848594961d
+size 180524

database/audio/sample_clips/english_fleurs_1512.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c30bddf019ff7abf8f81f0d6a0fc018b186613a9b2d67c3f3df24ae255e0b6b1
+size 345644

database/audio/sample_clips/english_fleurs_1545.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee23300897063f00999fb4a5f3bfd7521152b596ad91c6bf4758be54a855c2ba
+size 245164

database/audio/sample_clips/english_fleurs_1546.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be0fb2731a5012de1902223605b2a5110828482b65aa48b869eebaf319d2cdaf
+size 391724

database/audio/sample_clips/english_fleurs_1578.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:933a4569cc229f58e56cf680fa4cd3c7660b1e73cf6ef734ed906699bb81b1f9
+size 265644

database/audio/sample_clips/english_fleurs_1595.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d736aa4815a6de4ecc1f2864c4e6f9c67906b8eb37e8df103a4107d95d257377
+size 320684

database/audio/sample_clips/english_fleurs_1637.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bad0d5ef9019d1c9cbbc4aac7abe0ea49f5b4fd7956f92a369f70bc4946ae5f
+size 1015724

database/audio/sample_clips/english_fleurs_1639.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78af7de3a38af8b7a368096b7047650fc387491601bfe48221806cf46d94ec92
+size 291884

database/audio/sample_clips/english_fleurs_1645.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15a992f941349e2a03fafca3d96ea1a2d86ffb6aadf8833369095b21423e6042
+size 218924

database/audio/sample_clips/hindi_fleurs_1524.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e00d95d250c2ba7a3f2186b26f82a126e80b61bb667bd0ee2af712287c620bef
+size 382124

database/audio/sample_clips/hindi_fleurs_1526.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1b686404e5dadd0cdadc91c20737c9d5846b0ceb981f533576f1c1ec1c49cd1
+size 453164

database/audio/sample_clips/hindi_fleurs_1540.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2053febb046fab7e62066d77ecefa597f80db86e02843517400d4192d2df270
+size 276524

database/audio/sample_clips/hindi_fleurs_1549.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:855c89aa564662385d32b05d35e28edc60a6d05974735320886f913c87c493ee
+size 506924

database/audio/sample_clips/hindi_fleurs_1560.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c24145d731abae2b8e1323ff08edd944f8a2a1ed05035ebf43770f8190f8f9f
+size 192044

database/audio/sample_clips/hindi_fleurs_1581.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c58925c65f89f1ddeba6055d59b27e37ae49bf1ca0233ae0a1d1747f60a3168
+size 224684

database/audio/sample_clips/hindi_fleurs_1609.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4871a032b62c48f1d286d0b977b8218856c1205360359f96a273304870007e7c
+size 303404

database/audio/sample_clips/hindi_fleurs_1620.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41801b96eb739e2f6a0850c7f9cc75adb36468a9a81613922996276ac4bd4f38
+size 610604

database/audio/sample_clips/hindi_fleurs_1641.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c81049dd7cb390aa3267410ac4e1e2e95622d6085366fe3ff088471324cd8a5
+size 361004

database/audio/sample_clips/hindi_fleurs_1645.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f1aee1a6596889197682524a5ad105141d38b54318ed03e1bf3580b64f36c39
+size 169004

database/audio/sample_clips/hinglish_hiacc_AD09072.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd452e54e2e1432586f68f05d2cd25d0411d83dafc8f4dac000ff4cb59bc9597
+size 512084

database/audio/sample_clips/hinglish_hiacc_AD22092.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77a8459e9696d11be084b1671b8bca53d0c5275f52f4eb2a1c834d94992a94fd
+size 227710

database/audio/sample_clips/hinglish_hiacc_AD23025.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:599e5b5ccd3c1dbf321963d158eda124e3e6ca2c582b03e1786474732da7ddcb
+size 104730

database/audio/sample_clips/hinglish_hiacc_AD36007.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8223b79e5de802db3adb45d4bf724c9eadaf79701d20c357b1a56ea189dccc0
+size 137886

database/audio/sample_clips/hinglish_hiacc_AD36064.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f90d8b109474071f39fd8767c50d4afb5d7b22dd83e91f0ece9c3edb707426d7
+size 121006

database/audio/sample_clips/hinglish_hiacc_AD40049.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5234def68e6e0823873e2dc63f2586a3986f2ddf4ecb53a2d6d31945198ee34e
+size 289802

database/audio/sample_clips/hinglish_hiacc_AD40103.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1dc4b1146f75f1540b9348c02b4277d205189c41ebf1a65c8ba3ce939fefa607
+size 193346

database/audio/sample_clips/hinglish_hiacc_AD40158.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96eeca3c3bd867f0b4f953843004127cc1d6f486dcc6d637a6740d4cdd29a0a9
+size 173452

database/audio/sample_clips/hinglish_hiacc_AD60096.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:778f8fe8a0fa53585b93e61d94270f34cc681297ca5da4d2f0c9fec53d8a6a3f
+size 153124

database/audio/sample_clips/manifest.json

@@ -0,0 +1,232 @@
+[
+  {
+    "filename": "english_fleurs_1554.wav",
+    "bucket": "english",
+    "source": "fleurs_en_us",
+    "duration_seconds": 2.7,
+    "transcript": "he built a wifi door bell he said"
+  },
+  {
+    "filename": "english_fleurs_1511.wav",
+    "bucket": "english",
+    "source": "fleurs_en_us",
+    "duration_seconds": 5.64,
+    "transcript": "soon officers equipped with riot gear entered the yard and cornered the inmates with tear gas"
+  },
+  {
+    "filename": "english_fleurs_1645.wav",
+    "bucket": "english",
+    "source": "fleurs_en_us",
+    "duration_seconds": 6.84,
+    "transcript": "its all-pervading power affected everyone from king to commoner"
+  },
+  {
+    "filename": "english_fleurs_1545.wav",
+    "bucket": "english",
+    "source": "fleurs_en_us",
+    "duration_seconds": 7.66,
+    "transcript": "a curry can be either dry or wet depending on the amount of liquid"
+  },
+  {
+    "filename": "english_fleurs_1578.wav",
+    "bucket": "english",
+    "source": "fleurs_en_us",
+    "duration_seconds": 8.3,
+    "transcript": "then lakkha singh took the lead in singing the bhajans"
+  },
+  {
+    "filename": "english_fleurs_1639.wav",
+    "bucket": "english",
+    "source": "fleurs_en_us",
+    "duration_seconds": 9.12,
+    "transcript": "since students are often the most critical audience the blog writer begins to strive to improve writing to avoid criticism"
+  },
+  {
+    "filename": "english_fleurs_1595.wav",
+    "bucket": "english",
+    "source": "fleurs_en_us",
+    "duration_seconds": 10.02,
+    "transcript": "the tibetan buddhism is based on the teachings of buddha but were extended by the mahayana path of love and by a lot of techniques from indian yoga"
+  },
+  {
+    "filename": "english_fleurs_1512.wav",
+    "bucket": "english",
+    "source": "fleurs_en_us",
+    "duration_seconds": 10.8,
+    "transcript": "this new environment has different resources and different competitors so the new population will need different features or adaptations to be a strong competitor than what they had needed before"
+  },
+  {
+    "filename": "english_fleurs_1546.wav",
+    "bucket": "english",
+    "source": "fleurs_en_us",
+    "duration_seconds": 12.24,
+    "transcript": "the feathers' structure suggests that they were not used in flight but rather for temperature regulation or display the researchers suggested that even though this is the tail of a young dinosaur the sample shows adult plumage and not a chick's down"
+  },
+  {
+    "filename": "english_fleurs_1637.wav",
+    "bucket": "english",
+    "source": "fleurs_en_us",
+    "duration_seconds": 31.74,
+    "transcript": "the terrified king louis xvi queen marie antoinette their two young children 11 year old marie therese and four year old louis-charles and the king's sister madam elizabeth on the 6th october 1789 were forced back to paris from versailles by a mob of market women"
+  },
+  {
+    "filename": "hindi_fleurs_1645.wav",
+    "bucket": "hindi",
+    "source": "fleurs_hi_in",
+    "duration_seconds": 5.28,
+    "transcript": "\u0907\u0938\u0915\u0940 \u0938\u0930\u094d\u0935\u0935\u094d\u092f\u093e\u092a\u0940 \u0936\u0915\u094d\u0924\u093f \u0928\u0947 \u0930\u093e\u091c\u093e \u0938\u0947 \u0932\u0947\u0915\u0930 \u0906\u092e \u0924\u0915 \u0938\u092d\u0940 \u0915\u094b \u092a\u094d\u0930\u092d\u093e\u0935\u093f\u0924 \u0915\u093f\u092f\u093e"
+  },
+  {
+    "filename": "hindi_fleurs_1560.wav",
+    "bucket": "hindi",
+    "source": "fleurs_hi_in",
+    "duration_seconds": 6.0,
+    "transcript": "\u091c\u0948\u0938\u0947 \u0939\u0940 \u0906\u092a \u0927\u093e\u0930\u093e \u0938\u0947 \u0928\u093f\u0915\u0932 \u0906\u0924\u0947 \u0939\u0948\u0902 \u092a\u0940\u091b\u0947 \u0915\u0940 \u0913\u0930 \u0924\u0948\u0930\u0928\u0947 \u092e\u0947\u0902 \u0915\u094b\u0908 \u0916\u093e\u0938 \u092e\u0941\u0936\u094d\u0915\u093f\u0932 \u0928\u0939\u0940\u0902 \u0939\u094b\u0924\u0940 \u0939\u0948"
+  },
+  {
+    "filename": "hindi_fleurs_1581.wav",
+    "bucket": "hindi",
+    "source": "fleurs_hi_in",
+    "duration_seconds": 7.02,
+    "transcript": "\u0905\u092e\u0947\u091c\u093c\u0928 \u0927\u0930\u0924\u0940 \u0915\u0940 \u0938\u092c\u0938\u0947 \u091a\u094c\u0921\u093c\u0940 \u0928\u0926\u0940 \u092d\u0940 \u0939\u0948 \u091c\u094b \u0915\u092d\u0940-\u0915\u092d\u0940 \u091b\u0939 \u092e\u0940\u0932 \u091c\u093f\u0924\u0928\u0940 \u091a\u094c\u0921\u093c\u0940 \u0939\u094b \u091c\u093e\u0924\u0940 \u0939\u0948"
+  },
+  {
+    "filename": "hindi_fleurs_1540.wav",
+    "bucket": "hindi",
+    "source": "fleurs_hi_in",
+    "duration_seconds": 8.64,
+    "transcript": "\u0938\u0948\u0926\u094d\u0927\u093e\u0902\u0924\u093f\u0915 \u0930\u0942\u092a \u0938\u0947 \u0924\u093f\u092c\u094d\u092c\u0924\u0940 \u092c\u094c\u0926\u094d\u0927 \u0927\u0930\u094d\u092e \u092c\u0939\u0941\u0924 \u0938\u0930\u0932 \u0939\u0948 \u0907\u0938\u092e\u0947\u0902 \u0915\u0941\u0902\u0921\u0932\u093f\u0928\u0940 \u092f\u094b\u0917 \u0927\u094d\u092f\u093e\u0928 \u0914\u0930 \u0938\u092d\u0940 \u0915\u094b \u092a\u094d\u0930\u0947\u092e \u0938\u0947 \u0917\u0932\u0947 \u0932\u0917\u093e\u0928\u0947 \u0935\u093e\u0932\u093e \u092e\u093e\u0930\u094d\u0917 \u0936\u093e\u092e\u093f\u0932 \u0939\u0948"
+  },
+  {
+    "filename": "hindi_fleurs_1609.wav",
+    "bucket": "hindi",
+    "source": "fleurs_hi_in",
+    "duration_seconds": 9.48,
+    "transcript": "\u0930\u0949\u092c\u093f\u0928 \u0909\u0925\u092a\u094d\u092a\u093e \u0928\u0947 \u0915\u0947\u0935\u0932 41 \u0917\u0947\u0902\u0926\u094b\u0902 \u092e\u0947\u0902 11 \u091a\u094c\u0915\u094b\u0902 \u0914\u0930 2 \u091b\u0915\u094d\u0915\u094b\u0902 \u0915\u0940 \u092e\u0926\u0926 \u0938\u0947 70 \u0930\u0928 \u092c\u0928\u093e \u0915\u0930 \u092a\u093e\u0930\u0940 \u0915\u093e \u0938\u0930\u094d\u0935\u094b\u091a\u094d\u091a \u0938\u094d\u0915\u094b\u0930 \u092c\u0928\u093e\u092f\u093e"
+  },
+  {
+    "filename": "hindi_fleurs_1641.wav",
+    "bucket": "hindi",
+    "source": "fleurs_hi_in",
+    "duration_seconds": 11.28,
+    "transcript": "\u0939\u093e\u0932\u093e\u0902\u0915\u093f \u091c\u0902\u0917\u0932 \u0915\u093e \u0905\u0930\u094d\u0925 \u0915\u0947\u0935\u0932 \u092e\u0948\u0902\u0917\u094d\u0930\u094b\u0935 \u0926\u0932\u0926\u0932 \u0928\u0939\u0940\u0902 \u0939\u0948\u0902 \u0935\u0947 \u0915\u0941\u091b \u0905\u0902\u0924\u093f\u092e \u0936\u0915\u094d\u0924\u093f\u0936\u093e\u0932\u0940 \u091c\u0902\u0917\u0932\u094b\u0902 \u092e\u0947\u0902 \u0938\u0947 \u0939\u0948\u0902 \u091c\u094b \u0915\u092d\u0940 \u0917\u0902\u0917\u093e \u092a\u0920\u093e\u0930 \u0915\u094b \u0915\u0935\u0930 \u0915\u0930\u0924\u0947 \u0925\u0947"
+  },
+  {
+    "filename": "hindi_fleurs_1524.wav",
+    "bucket": "hindi",
+    "source": "fleurs_hi_in",
+    "duration_seconds": 11.94,
+    "transcript": "\u092e\u0940\u091f\u094d\u0930\u093f\u0915 \u092a\u094d\u0930\u0923\u093e\u0932\u0940 \u0915\u0947 \u0907\u0938\u094d\u0924\u0947\u092e\u093e\u0932 \u0928\u093f\u0930\u092a\u0947\u0915\u094d\u0937\u0935\u093e\u0926 \u0938\u0947 \u0917\u0923\u0924\u0902\u0924\u094d\u0930\u0935\u093e\u0926 \u0930\u093e\u0937\u094d\u091f\u094d\u0930\u0935\u093e\u0926 \u092e\u0947\u0902 \u092c\u0926\u0932\u093e\u0935 \u0914\u0930 \u0926\u0947\u0936 \u0915\u093e \u092d\u0930\u094b\u0938\u093e \u0932\u094b\u0917\u094b\u0902 \u0938\u0947 \u0939\u0948 \u0915\u093f\u0938\u0940 \u090f\u0915 \u0936\u093e\u0938\u0915 \u0938\u0947 \u0928\u0939\u0940\u0902 \u0910\u0938\u0940 \u092c\u093e\u0924\u094b\u0902 \u0915\u093e \u092c\u0939\u0941\u0924 \u091c\u093c\u094d\u092f\u093e\u0926\u093e \u0938\u093e\u092e\u093e\u091c\u093f\u0915 \u0914\u0930 \u0930\u093e\u091c\u0928\u0948\u0924\u093f\u0915 \u0905\u0938\u0930 \u0939\u094b\u0924\u093e \u0939\u0948"
+  },
+  {
+    "filename": "hindi_fleurs_1526.wav",
+    "bucket": "hindi",
+    "source": "fleurs_hi_in",
+    "duration_seconds": 14.16,
+    "transcript": "\u092f\u0926\u093f \u0906\u092a \u0938\u0930\u094d\u0926\u093f\u092f\u094b\u0902 \u092e\u0947\u0902 \u0906\u0930\u094d\u0915\u091f\u093f\u0915 \u092f\u093e \u0905\u0902\u091f\u093e\u0930\u094d\u0915\u091f\u093f\u0915 \u0915\u094d\u0937\u0947\u0924\u094d\u0930\u094b\u0902 \u092e\u0947\u0902 \u091c\u093e\u0924\u0947 \u0939\u0948\u0902 \u0924\u094b \u0906\u092a \u0927\u094d\u0930\u0941\u0935\u0940\u092f \u0930\u093e\u0924 \u0915\u093e \u0905\u0928\u0941\u092d\u0935 \u0915\u0930\u0947\u0902\u0917\u0947 \u091c\u093f\u0938\u0915\u093e \u0905\u0930\u094d\u0925 \u0939\u0948 \u0915\u093f \u0938\u0942\u0930\u091c \u0915\u094d\u0937\u093f\u0924\u093f\u091c \u0938\u0947 \u090a\u092a\u0930 \u0928\u0939\u0940\u0902 \u0909\u0917\u0924\u093e \u0939\u0948"
+  },
+  {
+    "filename": "hindi_fleurs_1549.wav",
+    "bucket": "hindi",
+    "source": "fleurs_hi_in",
+    "duration_seconds": 15.84,
+    "transcript": "\u0907\u0928 \u0938\u093f\u0926\u094d\u0927\u093e\u0902\u0924\u094b\u0902 \u0938\u0947 \u0938\u0902\u0915\u0947\u0924 \u092e\u093f\u0932\u0924\u093e \u0939\u0948 \u0915\u093f \u0932\u094b\u0917\u094b\u0902 \u0915\u0940 \u0915\u0941\u091b \u0910\u0938\u0940 \u091c\u093c\u0930\u0942\u0930\u0924\u0947\u0902 \u0914\u0930/\u092f\u093e \u0907\u091a\u094d\u091b\u093e\u090f\u0901 \u0925\u0940\u0902 \u091c\u094b \u0909\u0928\u0915\u0947 \u0935\u092f\u0938\u094d\u0915 \u0939\u094b\u0924\u0947-\u0939\u094b\u0924\u0947 \u0909\u0928\u0915\u0947 \u092d\u0940\u0924\u0930 \u0928\u093f\u0939\u093f\u0924 \u0939\u094b \u0917\u092f\u0940"
+  },
+  {
+    "filename": "hindi_fleurs_1620.wav",
+    "bucket": "hindi",
+    "source": "fleurs_hi_in",
+    "duration_seconds": 19.08,
+    "transcript": "\u091c\u093e\u0935\u093e \u0915\u0947 \u0935\u094d\u092f\u0902\u091c\u0928 \u0905\u092c \u092a\u0942\u0930\u0947 \u0926\u094d\u0935\u0940\u092a\u0938\u092e\u0942\u0939 \u092e\u0947\u0902 \u0935\u094d\u092f\u093e\u092a\u0915 \u0930\u0942\u092a \u0938\u0947 \u0909\u092a\u0932\u092c\u094d\u0927 \u0939\u0948\u0902 \u091c\u093f\u0938\u092e\u0947\u0902 \u092a\u093e\u0930\u0902\u092a\u0930\u093f\u0915 \u0935\u094d\u092f\u0902\u091c\u0928\u094b\u0902 \u0915\u0940 \u090f\u0915 \u0935\u093f\u0936\u0947\u0937\u0924\u093e \u0939\u0948 \u091c\u093e\u0935\u093e \u0915\u0947 \u0935\u094d\u092f\u0902\u091c\u0928\u094b\u0902 \u092e\u0947\u0902 \u092a\u094d\u0930\u092e\u0941\u0916 \u0938\u094d\u0935\u093e\u0926 \u092e\u0942\u0902\u0917\u092b\u0932\u0940 \u092e\u093f\u0930\u094d\u091a \u091a\u0940\u0928\u0940 \u0935\u093f\u0936\u0947\u0937 \u0930\u0942\u092a \u0938\u0947 \u091c\u093e\u0935\u093e \u0915\u0940 \u0928\u093e\u0930\u093f\u092f\u0932 \u091a\u0940\u0928\u0940 \u0914\u0930 \u0935\u093f\u092d\u093f\u0928\u094d\u0928 \u0938\u0941\u0917\u0902\u0927\u093f\u0924 \u092e\u0938\u093e\u0932\u094b\u0902 \u0915\u093e \u0939\u094b\u0924\u093e \u0939\u0948\u0964"
+  },
+  {
+    "filename": "hinglish_hiacc_AD13003.wav",
+    "bucket": "hinglish",
+    "source": "hiacc_adult_test",
+    "duration_seconds": 3.02,
+    "transcript": "So \u092e\u0947\u0930\u093e favourite festival \u0939\u0948 Diwali",
+    "code_mixing_index": 33.33,
+    "code_switch_count": 4
+  },
+  {
+    "filename": "hinglish_hiacc_AD23025.wav",
+    "bucket": "hinglish",
+    "source": "hiacc_adult_test",
+    "duration_seconds": 3.27,
+    "transcript": "\u0914\u0930 \u0909\u0928\u0915\u0940 friendship \u092c\u093f\u0932\u094d\u0915\u0941\u0932 \u091a\u0941\u091f\u0915\u093f\u092f\u094b\u0902 \u0915\u0940 \u0924\u0930\u0939 \u0925\u0940",
+    "code_mixing_index": 12.5,
+    "code_switch_count": 2
+  },
+  {
+    "filename": "hinglish_hiacc_AD36064.wav",
+    "bucket": "hinglish",
+    "source": "hiacc_adult_test",
+    "duration_seconds": 3.78,
+    "transcript": "\u0935\u094b as a memorial \u092c\u0928\u093e \u0939\u0948 those people who lost",
+    "code_mixing_index": 30.0,
+    "code_switch_count": 3
+  },
+  {
+    "filename": "hinglish_hiacc_AD36007.wav",
+    "bucket": "hinglish",
+    "source": "hiacc_adult_test",
+    "duration_seconds": 4.31,
+    "transcript": "Major reason is because  \u092e\u0941\u091d\u0947 lights \u092c\u0939\u0941\u0924 \u092a\u0938\u0902\u0926 \u0939\u0948 I love all these \u0926\u093f\u092f\u093e",
+    "code_mixing_index": 35.71,
+    "code_switch_count": 5
+  },
+  {
+    "filename": "hinglish_hiacc_AD60096.wav",
+    "bucket": "hinglish",
+    "source": "hiacc_adult_test",
+    "duration_seconds": 4.78,
+    "transcript": "\u0907\u0938\u0947 \u0938\u0941\u0928\u0915\u0947 \u0926\u094b\u0928\u094b\u0902 \u0928\u0947 \u0938\u094b\u091a\u093e \u0915\u093f \u092f\u0947 \u0909\u0928\u0915\u093e next adventure spot \u0939\u094b \u0938\u0915\u0924\u093e \u0939\u0948\u0964",
+    "code_mixing_index": 21.43,
+    "code_switch_count": 2
+  },
+  {
+    "filename": "hinglish_hiacc_AD40158.wav",
+    "bucket": "hinglish",
+    "source": "hiacc_adult_test",
+    "duration_seconds": 5.42,
+    "transcript": "\u0910\u0938\u093e \u0932\u0917 \u0930\u0939\u093e \u0939\u0948 \u091c\u0948\u0938\u0947 \u0915\u0940 superman \u092c\u0928\u0928\u0947 \u0915\u0940 \u0915\u094b\u0936\u093f\u0936 \u0915\u0930 \u0930\u0939\u093e \u0939\u0948 \u0924\u094b \u0907\u0938 image \u0938\u0947 \u092f\u0947 \u092d\u0940 \u0915\u0939\u093e \u091c\u093e \u0938\u0915\u0924\u093e \u0939\u0948 \u0915\u093f",
+    "code_mixing_index": 8.33,
+    "code_switch_count": 4
+  },
+  {
+    "filename": "hinglish_hiacc_AD40103.wav",
+    "bucket": "hinglish",
+    "source": "hiacc_adult_test",
+    "duration_seconds": 6.04,
+    "transcript": "\u091a\u093f\u0902\u091f\u0942 \u0914\u0930 \u0930\u093e\u091c\u0942 \u0928\u0947 guardian \u0915\u0947 \u0926\u093f\u090f \u0917\u090f task complete \u0915\u0930 \u0932\u093f\u090f \u0914\u0930 \u0939\u0930 \u090f\u0915 task \u0915\u0947 \u092c\u093e\u0926",
+    "code_mixing_index": 22.22,
+    "code_switch_count": 6
+  },
+  {
+    "filename": "hinglish_hiacc_AD22092.wav",
+    "bucket": "hinglish",
+    "source": "hiacc_adult_test",
+    "duration_seconds": 7.11,
+    "transcript": "\u0905\u092a\u0928\u093e \u0916\u093e\u0928\u093e \u0926\u0947\u0916 \u0915\u0947 \u0915\u094d\u092f\u0942\u0901\u0915\u093f \u090f\u0915 \u0915\u094b \u0915\u092e \u092e\u093f\u0932\u093e \u090f\u0915 \u0915\u094b \u091c\u093e\u0926\u093e \u092e\u093f\u0932\u093e \u0924\u092c  \u092d\u0940 \u0935\u094b unsatisfied \u0939\u0948 \u0909\u0938\u0915\u0947 dressing sense \u0914\u0930 \u0935\u094b \u0938\u092c \u091a\u0940\u091c\u0947 \u092d\u0940 \u0905\u091a\u094d\u091b\u0940 \u0939\u0948\u0964",
+    "code_mixing_index": 10.71,
+    "code_switch_count": 4
+  },
+  {
+    "filename": "hinglish_hiacc_AD40049.wav",
+    "bucket": "hinglish",
+    "source": "hiacc_adult_test",
+    "duration_seconds": 9.05,
+    "transcript": "\u092a\u0939\u093e\u0921\u0940 \u092f\u093e \u092b\u093f\u0930 \u0915\u093f\u0938\u0940 outstation \u091c\u0948\u0938\u0947 \u0936\u0939\u0930 \u092e\u0947\u0902 \u0939\u094b \u0938\u0915\u0924\u093e \u0939\u0948 \u0915\u094d\u092f\u0942\u0915\u093f \u0935\u0939\u093e\u0901 \u092a\u0947 \u092c\u0939\u0941\u0924 different \u091c\u0917\u0939 \u0915\u0947 \u0932\u094b\u0917 \u0939\u094b\u0924\u0947 \u0939\u0948\u0902\u0964",
+    "code_mixing_index": 9.52,
+    "code_switch_count": 4
+  },
+  {
+    "filename": "hinglish_hiacc_AD09072.wav",
+    "bucket": "hinglish",
+    "source": "hiacc_adult_test",
+    "duration_seconds": 16.0,
+    "transcript": "that's a barren land Barren \u092a\u0921\u093c\u0940 \u0939\u0948 \u0910\u0938\u093e \u0932\u0917\u0924\u093e \u0939\u0948 \u0915\u093f \u092f\u0947 desert \u0939\u0948 \u090f\u0915 \u0914\u0930 there are small small Mountains are there \u091c\u094b mountains \u0939\u0948 and that are short heighted mountain \u091c\u093f\u0938\u0938\u0947 \u0915\u0940 \u0938\u093e\u092b \u0928\u091c\u0930 \u0906\u0924\u093e \u0939\u0948 \u0915\u093f \u092f\u0947 \u090f\u0915 desert \u0915\u093e part \u0939\u0948",
+    "code_mixing_index": 48.89,
+    "code_switch_count": 13
+  }
+]