diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..62a5b0643cfb82436987fd9513645269097dacd7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,31 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/english_fleurs_1511.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/english_fleurs_1512.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/english_fleurs_1545.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/english_fleurs_1546.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/english_fleurs_1578.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/english_fleurs_1595.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/english_fleurs_1637.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/english_fleurs_1639.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/english_fleurs_1645.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hindi_fleurs_1524.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hindi_fleurs_1526.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hindi_fleurs_1540.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hindi_fleurs_1549.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hindi_fleurs_1560.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hindi_fleurs_1581.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hindi_fleurs_1609.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hindi_fleurs_1620.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hindi_fleurs_1641.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hindi_fleurs_1645.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hinglish_hiacc_AD09072.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hinglish_hiacc_AD22092.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hinglish_hiacc_AD23025.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hinglish_hiacc_AD36007.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hinglish_hiacc_AD36064.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hinglish_hiacc_AD40049.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hinglish_hiacc_AD40103.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hinglish_hiacc_AD40158.wav filter=lfs diff=lfs merge=lfs -text +database/audio/sample_clips/hinglish_hiacc_AD60096.wav filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index 3e853646498a25ebf5f047794a1d9ebdfbc48d51..369f3aaa664eaac6995f371f7c2c9bd8e37175b7 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,32 @@ --- -title: Turn Detection -emoji: ๐ŸŒ– -colorFrom: yellow -colorTo: blue +title: Turn Detection Live Dashboard +emoji: ๐ŸŽ™๏ธ +colorFrom: blue +colorTo: yellow sdk: gradio -sdk_version: 6.26.0 -python_version: '3.12' +sdk_version: 6.25.0 app_file: app.py pinned: false +license: mit --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +# Turn Detection โ€” Live Dashboard + +A real-time dashboard for audio turn detection: deciding whether a speaker is +finished talking or just pausing. Speak into the mic or replay a bundled clip +and watch several models score the decision live, plotted against the +waveform. + +## Models +Smart Turn v3.2 (zero-shot), Whisper with trained heads, a Qwen-based semantic +judge, the LiveKit end-of-turn detector, acoustic + semantic fusion, and a +streaming semantic branch. + +## How to use +1. Select one or more models. +2. Record from the mic, or replay a sample clip / upload a recording. +3. Each model's "turn complete" probability is drawn against the waveform; the + dashed line is the decision threshold. + +Trained head checkpoints and demo clips are bundled. Other model weights +download on first use. diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..5974534b98d032e6314cb96f9f6047b6fa544918 --- /dev/null +++ b/app.py @@ -0,0 +1,23 @@ +import traceback +import numpy as np +from backend.orchestrator import run_pipeline +from backend.presets import list_presets +from frontend.layout import build_app + +def _warmup() -> None: + sr = 16000 + dummy = np.zeros(sr, dtype=np.float32) + for preset in list_presets(): + if not preset.get('available'): + continue + try: + run_pipeline(preset['config'], dummy, sample_rate=sr, preset_label=preset['label']) + print(f"[warmup] ready: {preset['label']}") + except Exception as exc: + print(f"[warmup] skipped {preset['label']}: {exc}") + traceback.print_exc() + print('[warmup] done') +if __name__ == '__main__': + _warmup() + demo = build_app() + demo.launch(share=False) \ No newline at end of file diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/audio_utils.py b/backend/audio_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..cb80c5ea301d8e1736369f1510b9bbb6fa0edef0 --- /dev/null +++ b/backend/audio_utils.py @@ -0,0 +1,13 @@ +from __future__ import annotations +import numpy as np +import config + +def truncate_or_left_pad(audio: np.ndarray, n_seconds: int=config.SMART_TURN_WINDOW_SECONDS, sample_rate: int=config.SAMPLE_RATE) -> np.ndarray: + audio = np.asarray(audio, dtype=np.float32) + max_samples = n_seconds * sample_rate + if len(audio) > max_samples: + return audio[-max_samples:] + if len(audio) < max_samples: + padding = max_samples - len(audio) + return np.pad(audio, (padding, 0), mode='constant', constant_values=0) + return audio \ No newline at end of file diff --git a/backend/capabilities.py b/backend/capabilities.py new file mode 100644 index 0000000000000000000000000000000000000000..a67eb4b936cca9f2d6f857625e81d78e17bef4ef --- /dev/null +++ b/backend/capabilities.py @@ -0,0 +1,64 @@ +from __future__ import annotations +import json +from dataclasses import dataclass +from typing import Optional +import config +STATIC_CAPABILITIES: dict[str, dict] = {'gate.webrtcvad': {'available': True, 'reason': None, 'provenance': 'real_checkpoint'}, 'gate.silero_vad': {'available': True, 'reason': None, 'provenance': 'real_checkpoint'}, 'gate.none': {'available': True, 'reason': None, 'provenance': 'rule'}, 'acoustic.smart_turn_onnx': {'available': True, 'reason': None, 'provenance': 'real_checkpoint'}, 'semantic.qwen_local': {'available': True, 'reason': None, 'provenance': 'real_checkpoint'}, 'semantic.livekit_eou': {'available': True, 'reason': None, 'provenance': 'real_checkpoint'}, 'semantic.qwen_local_streaming': {'available': True, 'reason': None, 'provenance': 'architecture_reimplemented'}, 'semantic.groq_api': {'available': False, 'reason': 'no API key configured - set GROQ_API_KEY to enable (see docs/decision-log.md #17)', 'provenance': 'unavailable'}, 'semantic.openrouter_api': {'available': False, 'reason': 'no API key configured - set OPENROUTER_API_KEY to enable (see docs/decision-log.md #17)', 'provenance': 'unavailable'}, 'fusion.weighted_vote': {'available': True, 'reason': None, 'provenance': 'rule'}, 'fusion.easy_turn': {'available': False, 'reason': "Easy Turn's linguistic branch needs its own ASR component we haven't wired yet - checkpoint downloaded for reference only (see docs/decision-log.md #19)", 'provenance': 'unavailable'}, 'mode.full_duplex_bypass.moshi': {'available': False, 'reason': 'needs GPU VRAM not available on this machine; API routing deferred (see docs/decision-log.md #6, #17)', 'provenance': 'unavailable'}, 'mode.full_duplex_bypass.human1': {'available': False, 'reason': 'needs GPU VRAM not available on this machine; API routing deferred (see docs/decision-log.md #6, #17)', 'provenance': 'unavailable'}} + +@dataclass +class CapabilityInfo: + key: str + available: bool + reason: Optional[str] + provenance: Optional[str] + +def _head_checkpoint_path(encoder: str, pooling: str, head: str) -> tuple: + stem = f'{encoder}_{pooling}_{head}' + ckpt = config.CHECKPOINTS_DIR / f'{stem}.pt' + meta = config.CHECKPOINTS_DIR / f'{stem}.metadata.json' + return (ckpt, meta) + +def head_capability_key(encoder: str, pooling: str, head: str) -> str: + return f'acoustic.head.{encoder}.{pooling}.{head}' + +def _check_trained_head(encoder: str, pooling: str, head: str) -> CapabilityInfo: + key = head_capability_key(encoder, pooling, head) + ckpt, meta = _head_checkpoint_path(encoder, pooling, head) + if not (ckpt.exists() and meta.exists()): + return CapabilityInfo(key=key, available=False, reason=f'not trained yet - run experiments/train_head.py to produce database/checkpoints/{encoder}_{pooling}_{head}.pt', provenance='unavailable') + try: + metadata = json.loads(meta.read_text()) + except (OSError, json.JSONDecodeError) as exc: + return CapabilityInfo(key=key, available=False, reason=f'checkpoint metadata unreadable ({exc}) - retrain via experiments/train_head.py', provenance='unavailable') + if metadata.get('encoder') != encoder or metadata.get('pooling') != pooling or metadata.get('head') != head: + return CapabilityInfo(key=key, available=False, reason='checkpoint metadata does not match the requested (encoder, pooling, head) combination', provenance='unavailable') + return CapabilityInfo(key=key, available=True, reason=None, provenance='trained_by_us') + +def get(key: str) -> CapabilityInfo: + if key.startswith('acoustic.head.'): + _, _, encoder, pooling, head = key.split('.') + return _check_trained_head(encoder, pooling, head) + if key in STATIC_CAPABILITIES: + entry = STATIC_CAPABILITIES[key] + return CapabilityInfo(key=key, **entry) + raise KeyError(f'unknown capability key: {key!r}') + +def is_available(key: str) -> bool: + return get(key).available + +def list_trained_heads() -> list[CapabilityInfo]: + found = [] + if not config.CHECKPOINTS_DIR.exists(): + return found + for meta_path in sorted(config.CHECKPOINTS_DIR.glob('*.metadata.json')): + try: + metadata = json.loads(meta_path.read_text()) + except (OSError, json.JSONDecodeError): + continue + encoder, pooling, head = (metadata.get('encoder'), metadata.get('pooling'), metadata.get('head')) + if not all([encoder, pooling, head]): + continue + info = _check_trained_head(encoder, pooling, head) + if info.available: + found.append(info) + return found \ No newline at end of file diff --git a/backend/easy_turn.py b/backend/easy_turn.py new file mode 100644 index 0000000000000000000000000000000000000000..519f0543ac3625152d9d7c133477f039ce5ca15b --- /dev/null +++ b/backend/easy_turn.py @@ -0,0 +1,7 @@ +from __future__ import annotations +from backend.capabilities import get +from backend.types import StageResult + +def run(*_args, **_kwargs) -> StageResult: + info = get('fusion.easy_turn') + return StageResult(stage='fusion.easy_turn', timing_ms=0.0, output=None, available=False, reason=info.reason, provenance='unavailable') \ No newline at end of file diff --git a/backend/encoders.py b/backend/encoders.py new file mode 100644 index 0000000000000000000000000000000000000000..da8f85a32ebaa2964eb4564ee25685c408574f56 --- /dev/null +++ b/backend/encoders.py @@ -0,0 +1,79 @@ +from __future__ import annotations +import math +import time +from typing import Literal +import numpy as np +import torch +import config +from backend.audio_utils import truncate_or_left_pad +from backend.types import StageResult +EncoderName = Literal['whisper_tiny', 'whisper_base', 'wav2vec2'] +ENCODER_WINDOW_SECONDS = {'whisper_tiny': 30, 'whisper_base': 30, 'wav2vec2': config.SMART_TURN_WINDOW_SECONDS} +_ENCODER_OUTPUT_HOP_SAMPLES = 320 +_device = 'mps' if torch.backends.mps.is_available() else 'cpu' +_whisper_models: dict[str, tuple] = {} +_wav2vec2_model = None +_wav2vec2_processor = None + +def _get_whisper(model_id: str): + if model_id not in _whisper_models: + from transformers import WhisperFeatureExtractor, WhisperModel + feature_extractor = WhisperFeatureExtractor(chunk_length=30) + model = WhisperModel.from_pretrained(model_id).to(_device).eval() + _whisper_models[model_id] = (feature_extractor, model) + return _whisper_models[model_id] + +def _get_wav2vec2(): + global _wav2vec2_model, _wav2vec2_processor + if _wav2vec2_model is None: + from transformers import Wav2Vec2FeatureExtractor, Wav2Vec2Model + _wav2vec2_processor = Wav2Vec2FeatureExtractor.from_pretrained(config.WAV2VEC2_ID) + _wav2vec2_model = Wav2Vec2Model.from_pretrained(config.WAV2VEC2_ID).to(_device).eval() + return (_wav2vec2_processor, _wav2vec2_model) + +def _valid_output_frames(original_num_samples: int, window_seconds: int, sample_rate: int) -> int: + real_samples = min(original_num_samples, window_seconds * sample_rate) + return max(1, math.ceil(real_samples / _ENCODER_OUTPUT_HOP_SAMPLES)) + +@torch.inference_mode() +def _encode_whisper(model_id: str, audio: np.ndarray, sample_rate: int) -> tuple[np.ndarray, int]: + feature_extractor, model = _get_whisper(model_id) + window_seconds = ENCODER_WINDOW_SECONDS['whisper_tiny'] + window_samples = window_seconds * sample_rate + valid_frames = _valid_output_frames(len(audio), window_seconds, sample_rate) + padded = truncate_or_left_pad(audio, window_seconds, sample_rate) + inputs = feature_extractor(padded, sampling_rate=sample_rate, return_tensors='pt', padding='max_length', max_length=window_samples, truncation=True, do_normalize=True) + input_features = inputs.input_features.to(_device) + encoder = model.get_encoder() + hidden_states = encoder(input_features).last_hidden_state + hidden_states = hidden_states.squeeze(0).float().cpu().numpy() + valid_frames = min(valid_frames, hidden_states.shape[0]) + return (hidden_states, valid_frames) + +@torch.inference_mode() +def _encode_wav2vec2(audio: np.ndarray, sample_rate: int) -> tuple[np.ndarray, int]: + processor, model = _get_wav2vec2() + window_seconds = ENCODER_WINDOW_SECONDS['wav2vec2'] + valid_frames = _valid_output_frames(len(audio), window_seconds, sample_rate) + padded = truncate_or_left_pad(audio, window_seconds, sample_rate) + inputs = processor(padded, sampling_rate=sample_rate, return_tensors='pt') + input_values = inputs.input_values.to(_device) + hidden_states = model(input_values).last_hidden_state + hidden_states = hidden_states.squeeze(0).float().cpu().numpy() + valid_frames = min(valid_frames, hidden_states.shape[0]) + return (hidden_states, valid_frames) + +def extract_hidden_states(encoder: EncoderName, audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> tuple[np.ndarray, int]: + if encoder == 'whisper_tiny': + return _encode_whisper(config.WHISPER_TINY_ID, audio, sample_rate) + if encoder == 'whisper_base': + return _encode_whisper(config.WHISPER_BASE_ID, audio, sample_rate) + if encoder == 'wav2vec2': + return _encode_wav2vec2(audio, sample_rate) + raise ValueError(f'unknown encoder: {encoder!r}') + +def run(encoder: EncoderName, audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> StageResult: + start = time.perf_counter() + hidden_states, valid_length = extract_hidden_states(encoder, audio, sample_rate) + timing_ms = (time.perf_counter() - start) * 1000 + return StageResult(stage=f'encoder.{encoder}', timing_ms=timing_ms, output={'hidden_states': hidden_states, 'valid_length': valid_length}, available=True, provenance='real_checkpoint') \ No newline at end of file diff --git a/backend/fusion.py b/backend/fusion.py new file mode 100644 index 0000000000000000000000000000000000000000..9269b72224a3c46ed62f4d743b9e5a317324c14c --- /dev/null +++ b/backend/fusion.py @@ -0,0 +1,21 @@ +from __future__ import annotations +import time +from typing import Optional +from backend.types import StageResult + +def weighted_vote(acoustic_probability: float, semantic_verdict: Optional[str]=None, semantic_probability: Optional[float]=None, acoustic_weight: float=0.6) -> dict: + if semantic_verdict == 'wait': + return {'decision': 'incomplete', 'probability': min(acoustic_probability, 0.2), 'rule': "semantic 'wait' verdict overrides acoustic signal"} + if semantic_probability is not None: + semantic_score = semantic_probability + else: + semantic_score = 1.0 if semantic_verdict == 'complete' else 0.0 + fused_probability = acoustic_weight * acoustic_probability + (1 - acoustic_weight) * semantic_score + decision = 'complete' if fused_probability > 0.5 else 'incomplete' + return {'decision': decision, 'probability': fused_probability, 'rule': f'weighted_vote(acoustic_weight={acoustic_weight})'} + +def run(acoustic_probability: float, semantic_verdict: Optional[str]=None, semantic_probability: Optional[float]=None, acoustic_weight: float=0.6) -> StageResult: + start = time.perf_counter() + result = weighted_vote(acoustic_probability, semantic_verdict, semantic_probability, acoustic_weight) + timing_ms = (time.perf_counter() - start) * 1000 + return StageResult(stage='fusion.weighted_vote', timing_ms=timing_ms, output=result, available=True, provenance='rule') \ No newline at end of file diff --git a/backend/gate.py b/backend/gate.py new file mode 100644 index 0000000000000000000000000000000000000000..20f465bfc34448ab14c84bfda5a8bf5986cd3528 --- /dev/null +++ b/backend/gate.py @@ -0,0 +1,67 @@ +from __future__ import annotations +import time +import numpy as np +import webrtcvad +import config +from backend.types import StageResult +_silero_model = None +_silero_utils = None + +def _get_silero(): + global _silero_model, _silero_utils + if _silero_model is None: + import torch + _silero_model, _silero_utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', model='silero_vad', trust_repo=True) + return (_silero_model, _silero_utils) + +def _float_audio(audio: np.ndarray) -> np.ndarray: + audio = np.asarray(audio) + if audio.dtype.kind == 'i': + return audio.astype(np.float32) / 32768.0 + return audio.astype(np.float32) + +def _pcm16_bytes(audio_f32: np.ndarray) -> bytes: + clipped = np.clip(audio_f32, -1.0, 1.0) + return (clipped * 32767.0).astype(np.int16).tobytes() + +def run_webrtcvad(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE, aggressiveness: int=2) -> StageResult: + start = time.perf_counter() + audio_f32 = _float_audio(audio) + pcm = _pcm16_bytes(audio_f32) + frame_ms = 30 + frame_bytes = int(sample_rate * (frame_ms / 1000.0)) * 2 + vad = webrtcvad.Vad(aggressiveness) + speech_frames = 0 + total_frames = 0 + for offset in range(0, len(pcm) - frame_bytes + 1, frame_bytes): + frame = pcm[offset:offset + frame_bytes] + total_frames += 1 + if vad.is_speech(frame, sample_rate): + speech_frames += 1 + speech_detected = total_frames > 0 and speech_frames / total_frames > 0.1 + timing_ms = (time.perf_counter() - start) * 1000 + return StageResult(stage='gate.webrtcvad', timing_ms=timing_ms, output={'speech_detected': speech_detected, 'speech_frame_ratio': speech_frames / total_frames if total_frames else 0.0}, available=True, provenance='real_checkpoint') + +def run_silero(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE, threshold: float=0.5) -> StageResult: + import torch + start = time.perf_counter() + model, utils = _get_silero() + get_speech_timestamps = utils[0] + audio_f32 = _float_audio(audio) + tensor = torch.from_numpy(audio_f32) + timestamps = get_speech_timestamps(tensor, model, sampling_rate=sample_rate, threshold=threshold) + speech_detected = len(timestamps) > 0 + timing_ms = (time.perf_counter() - start) * 1000 + return StageResult(stage='gate.silero_vad', timing_ms=timing_ms, output={'speech_detected': speech_detected, 'speech_segments': timestamps}, available=True, provenance='real_checkpoint') + +def run_none(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> StageResult: + return StageResult(stage='gate.none', timing_ms=0.0, output={'speech_detected': True}, available=True, provenance='rule') + +def run(gate: str, audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE, **kwargs) -> StageResult: + if gate == 'webrtcvad': + return run_webrtcvad(audio, sample_rate, aggressiveness=kwargs.get('vad_aggressiveness', 2)) + if gate == 'silero_vad': + return run_silero(audio, sample_rate) + if gate == 'none': + return run_none(audio, sample_rate) + raise ValueError(f'unknown gate: {gate!r}') \ No newline at end of file diff --git a/backend/heads.py b/backend/heads.py new file mode 100644 index 0000000000000000000000000000000000000000..25b6a51032184ef09364a5434d27e21a6e6b43c2 --- /dev/null +++ b/backend/heads.py @@ -0,0 +1,56 @@ +from __future__ import annotations +import json +from pathlib import Path +from typing import Optional +import torch +import torch.nn as nn +import config + +class LinearHead(nn.Module): + + def __init__(self, input_dim: int): + super().__init__() + self.linear = nn.Linear(input_dim, 1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(x).squeeze(-1) + +class MLPHead(nn.Module): + + def __init__(self, input_dim: int, hidden_dim: int=128): + super().__init__() + self.net = nn.Sequential(nn.Linear(input_dim, hidden_dim), nn.ReLU(), nn.Linear(hidden_dim, 1)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.net(x).squeeze(-1) + +def build_head(head_type: str, input_dim: int) -> nn.Module: + if head_type == 'linear': + return LinearHead(input_dim) + if head_type == 'mlp': + return MLPHead(input_dim) + raise ValueError(f'unknown head_type: {head_type!r}') + +def checkpoint_paths(encoder: str, pooling: str, head: str) -> tuple[Path, Path]: + stem = f'{encoder}_{pooling}_{head}' + return (config.CHECKPOINTS_DIR / f'{stem}.pt', config.CHECKPOINTS_DIR / f'{stem}.metadata.json') + +def save(encoder: str, pooling: str, head_type: str, input_dim: int, head_state_dict: dict, metrics: dict, pooling_state_dict: Optional[dict]=None, trained_on: Optional[str]=None) -> None: + config.CHECKPOINTS_DIR.mkdir(parents=True, exist_ok=True) + ckpt_path, meta_path = checkpoint_paths(encoder, pooling, head_type) + torch.save({'head_state_dict': head_state_dict, 'pooling_state_dict': pooling_state_dict, 'input_dim': input_dim}, ckpt_path) + metadata = {'encoder': encoder, 'pooling': pooling, 'head': head_type, 'input_dim': input_dim, 'metrics': metrics} + if trained_on is not None: + metadata['trained_on'] = trained_on + meta_path.write_text(json.dumps(metadata, indent=2)) + +def load(encoder: str, pooling: str, head_type: str) -> tuple[nn.Module, Optional[dict], dict]: + ckpt_path, meta_path = checkpoint_paths(encoder, pooling, head_type) + if not (ckpt_path.exists() and meta_path.exists()): + raise FileNotFoundError(f'no trained checkpoint at {ckpt_path} - run experiments/train_head.py first (see backend/capabilities.py for the exact command).') + metadata = json.loads(meta_path.read_text()) + blob = torch.load(ckpt_path, map_location='cpu', weights_only=True) + head = build_head(head_type, blob['input_dim']) + head.load_state_dict(blob['head_state_dict']) + head.eval() + return (head, blob.get('pooling_state_dict'), metadata) \ No newline at end of file diff --git a/backend/orchestrator.py b/backend/orchestrator.py new file mode 100644 index 0000000000000000000000000000000000000000..592ad1d9a8d957c30ffab08430f652079ae6f81d --- /dev/null +++ b/backend/orchestrator.py @@ -0,0 +1,117 @@ +from __future__ import annotations +import time +from typing import Optional +import numpy as np +import torch +import config as cfg +from backend import encoders, fusion, gate, heads, pooling, semantic, smart_turn_onnx, streaming_semantic +from backend.capabilities import get as get_capability +from backend.capabilities import head_capability_key +from backend.types import PipelineConfig, PipelineResult, StageResult, StageUnavailableError + +def _require(capability_key: str) -> None: + info = get_capability(capability_key) + if not info.available: + raise StageUnavailableError(f'{capability_key} is not available: {info.reason}') + +def _run_custom_acoustic_branch(pc: PipelineConfig, audio: np.ndarray, sample_rate: int) -> tuple[StageResult, StageResult]: + head_key = head_capability_key(pc.encoder, pc.pooling, pc.head) + _require(head_key) + encoder_result = encoders.run(pc.encoder, audio, sample_rate) + hidden_states = encoder_result.output['hidden_states'] + valid_length = encoder_result.output['valid_length'] + head_module, pooling_state_dict, metadata = heads.load(pc.encoder, pc.pooling, pc.head) + pool_start = time.perf_counter() + with torch.inference_mode(): + if pc.pooling == 'mean': + pooled = pooling.mean_pool(hidden_states, valid_length) + pooled_tensor = torch.from_numpy(pooled).float().unsqueeze(0) + elif pc.pooling == 'cross_attention': + attn_pool = pooling.CrossAttentionPool(hidden_dim=hidden_states.shape[-1]) + attn_pool.load_state_dict(pooling_state_dict) + attn_pool.eval() + hs_tensor = torch.from_numpy(hidden_states).float().unsqueeze(0) + vl_tensor = torch.tensor([valid_length]) + pooled_tensor = attn_pool(hs_tensor, vl_tensor) + else: + raise ValueError(f'unknown pooling: {pc.pooling!r}') + logit = head_module(pooled_tensor) + probability = torch.sigmoid(logit).item() + pool_head_ms = (time.perf_counter() - pool_start) * 1000 + decision = 'complete' if probability > 0.5 else 'incomplete' + head_result = StageResult(stage=f'acoustic.custom.{pc.encoder}.{pc.pooling}.{pc.head}', timing_ms=pool_head_ms, output={'decision': decision, 'probability': probability, 'metrics_at_train_time': metadata.get('metrics')}, available=True, provenance='trained_by_us') + return (encoder_result, head_result) + +def run_pipeline(pc: PipelineConfig, audio: np.ndarray, sample_rate: int=cfg.SAMPLE_RATE, preset_label: Optional[str]=None) -> PipelineResult: + if pc.full_duplex != 'off': + _require(f'mode.full_duplex_bypass.{pc.full_duplex}') + raise StageUnavailableError('full-duplex bypass is not implemented in this pass') + stage_results: list[StageResult] = [] + start = time.perf_counter() + _require(f'gate.{pc.gate}') + gate_result = gate.run(pc.gate, audio, sample_rate, vad_aggressiveness=pc.vad_aggressiveness) + stage_results.append(gate_result) + acoustic_result = None + acoustic_probability = None + acoustic_provenance = None + if pc.encoder == 'none': + pass + elif pc.encoder == 'smart_turn_onnx': + _require('acoustic.smart_turn_onnx') + acoustic_result = smart_turn_onnx.run(audio, sample_rate) + stage_results.append(acoustic_result) + acoustic_probability = acoustic_result.output['probability'] + acoustic_provenance = 'real_checkpoint' + else: + encoder_result, head_result = _run_custom_acoustic_branch(pc, audio, sample_rate) + stage_results.extend([encoder_result, head_result]) + acoustic_result = head_result + acoustic_probability = head_result.output['probability'] + acoustic_provenance = 'trained_by_us' + transcript = None + semantic_verdict = None + semantic_probability = None + if pc.semantic != 'off': + _require(f'semantic.{pc.semantic}') + if pc.semantic == 'qwen_local': + semantic_result = semantic.run(audio, sample_rate, temperature=pc.semantic_temperature) + stage_results.append(semantic_result) + transcript = semantic_result.output['transcript'] + semantic_verdict = semantic_result.output['verdict'] + elif pc.semantic == 'qwen_local_streaming': + semantic_result = streaming_semantic.run(audio, sample_rate, temperature=pc.semantic_temperature) + stage_results.append(semantic_result) + transcript = semantic_result.output['transcript'] + semantic_verdict = semantic_result.output['verdict'] + elif pc.semantic == 'livekit_eou': + semantic_result = semantic.run_livekit(audio, sample_rate) + stage_results.append(semantic_result) + transcript = semantic_result.output['transcript'] + semantic_probability = semantic_result.output['probability'] + else: + raise StageUnavailableError(f'semantic provider {pc.semantic!r} is not implemented yet') + if pc.fusion == 'weighted_vote': + _require('fusion.weighted_vote') + fusion_result = fusion.run(acoustic_probability, semantic_verdict, semantic_probability, pc.acoustic_weight) + stage_results.append(fusion_result) + decision = fusion_result.output['decision'] + probability = fusion_result.output['probability'] + result_provenance = 'rule' + elif pc.fusion == 'easy_turn': + _require('fusion.easy_turn') + raise StageUnavailableError('unreachable') + elif pc.encoder == 'none': + if semantic_probability is not None: + decision = 'complete' if semantic_probability > 0.5 else 'incomplete' + probability = semantic_probability + result_provenance = 'real_checkpoint' + else: + decision = 'complete' if semantic_verdict == 'complete' else 'incomplete' + probability = None + result_provenance = 'architecture_reimplemented' if pc.semantic == 'qwen_local_streaming' else 'real_checkpoint' + else: + decision = acoustic_result.output['decision'] + probability = acoustic_probability + result_provenance = acoustic_provenance + total_latency_ms = (time.perf_counter() - start) * 1000 + return PipelineResult(config=pc, decision=decision, probability=probability, transcript=transcript, semantic_verdict=semantic_verdict, stage_results=stage_results, total_latency_ms=total_latency_ms, provenance=result_provenance, preset_label=preset_label) \ No newline at end of file diff --git a/backend/pooling.py b/backend/pooling.py new file mode 100644 index 0000000000000000000000000000000000000000..e827a63ee5c7407aa73919ceeb4cc056aa5698d2 --- /dev/null +++ b/backend/pooling.py @@ -0,0 +1,26 @@ +from __future__ import annotations +import numpy as np +import torch +import torch.nn as nn + +def mean_pool(hidden_states: np.ndarray, valid_length: int) -> np.ndarray: + valid_length = max(1, min(valid_length, hidden_states.shape[0])) + return hidden_states[-valid_length:].mean(axis=0) + +class CrossAttentionPool(nn.Module): + + def __init__(self, hidden_dim: int): + super().__init__() + self.hidden_dim = hidden_dim + self.query = nn.Parameter(torch.randn(hidden_dim) * hidden_dim ** (-0.5)) + + def forward(self, hidden_states: torch.Tensor, valid_lengths: torch.Tensor) -> torch.Tensor: + batch, seq_len, dim = hidden_states.shape + positions = torch.arange(seq_len, device=hidden_states.device).unsqueeze(0) + start_idx = (seq_len - valid_lengths).unsqueeze(1) + mask = positions >= start_idx + scores = hidden_states @ self.query / dim ** 0.5 + scores = scores.masked_fill(~mask, float('-inf')) + weights = torch.softmax(scores, dim=-1) + pooled = torch.einsum('bt,btd->bd', weights, hidden_states) + return pooled \ No newline at end of file diff --git a/backend/presets.py b/backend/presets.py new file mode 100644 index 0000000000000000000000000000000000000000..00d72c7f7f375966761b5665062bb3752267943e --- /dev/null +++ b/backend/presets.py @@ -0,0 +1,46 @@ +from __future__ import annotations +from dataclasses import dataclass +from backend.capabilities import get as get_capability +from backend.capabilities import head_capability_key +from backend.types import PipelineConfig, Provenance + +@dataclass +class Preset: + label: str + config: PipelineConfig + approximates: str + static_provenance: Provenance + +def _required_capability_keys(pc: PipelineConfig) -> list[str]: + keys = [f'gate.{pc.gate}'] + if pc.encoder == 'smart_turn_onnx': + keys.append('acoustic.smart_turn_onnx') + elif pc.encoder != 'none': + keys.append(head_capability_key(pc.encoder, pc.pooling, pc.head)) + if pc.semantic != 'off': + keys.append(f'semantic.{pc.semantic}') + if pc.fusion != 'off': + keys.append(f'fusion.{pc.fusion}') + return keys +_PRESET_DEFS: list[Preset] = [Preset(label='Smart Turn v3.2 (zero-shot)', config=PipelineConfig(gate='silero_vad', encoder='smart_turn_onnx', semantic='off', fusion='off'), approximates='Smart Turn v3.2 (Pipecat)', static_provenance='real_checkpoint'), Preset(label='Whisper-Tiny + Mean-Pool + Linear (trained)', config=PipelineConfig(gate='silero_vad', encoder='whisper_tiny', pooling='mean', head='linear', semantic='off', fusion='off'), approximates="Smart Turn v2's recipe (mean pool + linear), retrained by us", static_provenance='trained_by_us'), Preset(label='Whisper-Tiny + Mean-Pool + MLP (trained)', config=PipelineConfig(gate='silero_vad', encoder='whisper_tiny', pooling='mean', head='mlp', semantic='off', fusion='off'), approximates='', static_provenance='trained_by_us'), Preset(label='Whisper-Tiny + Cross-Attn + Linear (trained)', config=PipelineConfig(gate='silero_vad', encoder='whisper_tiny', pooling='cross_attention', head='linear', semantic='off', fusion='off'), approximates="Smart Turn v3.2's recipe (cross-attn pool + linear), retrained by us on real Hindi/Hinglish audio (HiACC + FLEURS) after the same architecture scored worst OOD (1/5) trained on the challenge's synthetic TTS data - real data fixed it outright (5/5). See docs/decision-log.md #38, experiments/results/dataset_comparison.md.", static_provenance='trained_by_us'), Preset(label='TEN Turn Detection (prompted)', config=PipelineConfig(gate='silero_vad', encoder='none', semantic='qwen_local', fusion='off'), approximates='TEN Turn Detection - approximated with local Qwen2.5-0.5B instead of their 7B (docs/decision-log.md #11)', static_provenance='real_checkpoint'), Preset(label='LiveKit End-of-Turn Detector (zero-shot)', config=PipelineConfig(gate='silero_vad', encoder='none', semantic='livekit_eou', fusion='off'), approximates='livekit/turn-detector - a real checkpoint, not an approximation (docs/decision-log.md #33)', static_provenance='real_checkpoint'), Preset(label='Acoustic + Semantic Fusion (rule-based)', config=PipelineConfig(gate='silero_vad', encoder='whisper_tiny', pooling='mean', head='linear', semantic='livekit_eou', fusion='weighted_vote'), approximates="A zero-training combination of our trained acoustic head with LiveKit's real EOU probability (docs/decision-log.md #33) - NOT Easy Turn's actual jointly-trained fusion, and no 'wait' handling since LiveKit's model has no such class (traded away when this preset moved off qwen_local)", static_provenance='rule'), Preset(label='FastTurn Streaming (reimplemented)', config=PipelineConfig(gate='silero_vad', encoder='smart_turn_onnx', semantic='qwen_local_streaming', fusion='weighted_vote'), approximates="FastTurn's described streaming-fusion mechanism, reimplemented by us for Hindi/Hinglish with a CTC checkpoint (theainerd/Wav2Vec2-large-xlsr-hindi) - NOT their code or reported numbers, which were never released (docs/decision-log.md #23)", static_provenance='architecture_reimplemented'), Preset(label='Easy Turn (unavailable)', config=PipelineConfig(gate='silero_vad', encoder='whisper_tiny', pooling='mean', head='linear', semantic='qwen_local', fusion='easy_turn'), approximates="Easy Turn's real joint acoustic+linguistic fusion - NOT wired up this pass", static_provenance='unavailable')] + +def list_presets() -> list[dict]: + out = [] + for preset in _PRESET_DEFS: + try: + infos = [get_capability(key) for key in _required_capability_keys(preset.config)] + failing = [i for i in infos if not i.available] + if failing: + available = False + reason = '; '.join((f'{i.key}: {i.reason}' for i in failing)) + provenance = 'unavailable' + else: + available = True + reason = None + provenance = preset.static_provenance + except (KeyError, ValueError) as exc: + available = False + reason = f'preset is not runnable as configured: {exc}' + provenance = 'unavailable' + out.append({'label': preset.label, 'config': preset.config, 'approximates': preset.approximates, 'available': available, 'reason': reason, 'provenance': provenance}) + return out \ No newline at end of file diff --git a/backend/semantic.py b/backend/semantic.py new file mode 100644 index 0000000000000000000000000000000000000000..3c77904f1c2751147c97e5ef9dbe99afb61a1928 --- /dev/null +++ b/backend/semantic.py @@ -0,0 +1,97 @@ +from __future__ import annotations +import re +import time +import unicodedata +import numpy as np +import torch +import config +from backend.types import StageResult +_device = 'mps' if torch.backends.mps.is_available() else 'cpu' +_asr_pipeline = None +_qwen_model = None +_qwen_tokenizer = None +_livekit_model = None +_livekit_tokenizer = None +_livekit_im_end_id = None +PROMPT_TEMPLATE = 'You are analyzing a snippet of transcribed speech from a Hindi-English (Hinglish) conversation with a voice assistant. Code-switching between Hindi and English mid-sentence is normal and not a sign of incompleteness. Filler words like "matlab", "toh", "haan", "wo kya bolte hain", "um", "uh" indicate the speaker is still thinking and has NOT completed their turn.\n\nTranscript: "{transcript}"\n\nClassify whether the speaker\'s turn is:\n- complete: the utterance is a complete thought, the speaker is done\n- incomplete: the utterance is grammatically or semantically incomplete, more is coming\n- wait: the speaker is explicitly asking for a pause (e.g. "ek second", "hold on", "wait")\n\nRespond with exactly one word: complete, incomplete, or wait.' +_VALID_LABELS = ('complete', 'incomplete', 'wait') + +def _get_asr_pipeline(): + global _asr_pipeline + if _asr_pipeline is None: + from transformers import pipeline + _asr_pipeline = pipeline('automatic-speech-recognition', model=config.WHISPER_TINY_ID, device=_device if _device != 'mps' else -1) + return _asr_pipeline + +def _get_qwen(): + global _qwen_model, _qwen_tokenizer + if _qwen_model is None: + from transformers import AutoModelForCausalLM, AutoTokenizer + _qwen_tokenizer = AutoTokenizer.from_pretrained(config.QWEN_LOCAL_ID) + _qwen_model = AutoModelForCausalLM.from_pretrained(config.QWEN_LOCAL_ID).to(_device).eval() + return (_qwen_model, _qwen_tokenizer) + +def _get_livekit(): + global _livekit_model, _livekit_tokenizer, _livekit_im_end_id + if _livekit_model is None: + from transformers import AutoModelForCausalLM, AutoTokenizer + _livekit_tokenizer = AutoTokenizer.from_pretrained(config.LIVEKIT_TURN_DETECTOR_ID) + _livekit_model = AutoModelForCausalLM.from_pretrained(config.LIVEKIT_TURN_DETECTOR_ID).to(_device).eval() + _livekit_im_end_id = _livekit_tokenizer.convert_tokens_to_ids('<|im_end|>') + return (_livekit_model, _livekit_tokenizer, _livekit_im_end_id) +_PUNCTUATION_RE = re.compile("[^\\w\\s'-]", re.UNICODE) +_WHITESPACE_RE = re.compile('\\s+') + +def _normalize_for_livekit(transcript: str) -> str: + text = unicodedata.normalize('NFKC', transcript).lower() + text = _PUNCTUATION_RE.sub(' ', text) + return _WHITESPACE_RE.sub(' ', text).strip() + +def transcribe(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> str: + asr = _get_asr_pipeline() + result = asr({'raw': np.asarray(audio, dtype=np.float32), 'sampling_rate': sample_rate}) + return result['text'].strip() + +def _parse_label(raw_text: str) -> str: + lowered = raw_text.lower() + for label in _VALID_LABELS: + if re.search(f'\\b{label}\\b', lowered): + return label + return 'incomplete' + +@torch.inference_mode() +def classify_transcript(transcript: str, temperature: float=0.2) -> dict: + model, tokenizer = _get_qwen() + messages = [{'role': 'user', 'content': PROMPT_TEMPLATE.format(transcript=transcript)}] + prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + inputs = tokenizer(prompt, return_tensors='pt').to(_device) + do_sample = temperature > 0 + output_ids = model.generate(**inputs, max_new_tokens=8, do_sample=do_sample, temperature=temperature if do_sample else None, pad_token_id=tokenizer.eos_token_id) + generated = output_ids[0][inputs['input_ids'].shape[1]:] + raw_text = tokenizer.decode(generated, skip_special_tokens=True) + label = _parse_label(raw_text) + return {'verdict': label, 'raw_response': raw_text.strip()} + +def run(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE, temperature: float=0.2) -> StageResult: + start = time.perf_counter() + transcript = transcribe(audio, sample_rate) + result = classify_transcript(transcript, temperature) + timing_ms = (time.perf_counter() - start) * 1000 + return StageResult(stage='semantic.qwen_local', timing_ms=timing_ms, output={'transcript': transcript, 'verdict': result['verdict'], 'raw_response': result['raw_response']}, available=True, provenance='real_checkpoint') + +@torch.inference_mode() +def classify_transcript_livekit(transcript: str) -> float: + model, tokenizer, im_end_id = _get_livekit() + normalized = _normalize_for_livekit(transcript) + prompt = f'<|im_start|><|user|>{normalized}' + inputs = tokenizer(prompt, return_tensors='pt').to(_device) + logits = model(**inputs).logits[0, -1, :] + probs = torch.softmax(logits, dim=-1) + return probs[im_end_id].item() + +def run_livekit(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> StageResult: + start = time.perf_counter() + transcript = transcribe(audio, sample_rate) + probability = classify_transcript_livekit(transcript) + timing_ms = (time.perf_counter() - start) * 1000 + return StageResult(stage='semantic.livekit_eou', timing_ms=timing_ms, output={'transcript': transcript, 'probability': probability}, available=True, provenance='real_checkpoint') \ No newline at end of file diff --git a/backend/smart_turn_onnx.py b/backend/smart_turn_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..927644fca64a063ebc1c71071af5324e21c3e3da --- /dev/null +++ b/backend/smart_turn_onnx.py @@ -0,0 +1,50 @@ +from __future__ import annotations +import time +from pathlib import Path +import numpy as np +import onnxruntime as ort +from transformers import WhisperFeatureExtractor +import config +from backend.audio_utils import truncate_or_left_pad +from backend.types import StageResult +_session: ort.InferenceSession | None = None +_feature_extractor: WhisperFeatureExtractor | None = None + +def _onnx_path() -> Path: + return config.MODELS_DIR / config.SMART_TURN_ONNX_FILENAME + +def _get_session() -> ort.InferenceSession: + global _session + if _session is None: + path = _onnx_path() + if not path.exists(): + from huggingface_hub import hf_hub_download + config.MODELS_DIR.mkdir(parents=True, exist_ok=True) + hf_hub_download(repo_id=config.SMART_TURN_REPO, filename=config.SMART_TURN_ONNX_FILENAME, local_dir=config.MODELS_DIR) + _session = ort.InferenceSession(str(path)) + return _session + +def _get_feature_extractor() -> WhisperFeatureExtractor: + global _feature_extractor + if _feature_extractor is None: + _feature_extractor = WhisperFeatureExtractor(chunk_length=config.SMART_TURN_WINDOW_SECONDS) + return _feature_extractor + +def preprocess(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> np.ndarray: + if sample_rate != config.SAMPLE_RATE: + raise ValueError(f'Smart Turn expects {config.SAMPLE_RATE}Hz mono PCM, got sample_rate={sample_rate}. Resample before calling this function.') + audio = truncate_or_left_pad(audio, config.SMART_TURN_WINDOW_SECONDS, sample_rate) + fe = _get_feature_extractor() + inputs = fe(audio, sampling_rate=sample_rate, return_tensors='np', padding='max_length', max_length=config.SMART_TURN_WINDOW_SAMPLES, truncation=True, do_normalize=True) + input_features = inputs.input_features.squeeze(0).astype(np.float32) + return np.expand_dims(input_features, axis=0) + +def run(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> StageResult: + start = time.perf_counter() + session = _get_session() + input_features = preprocess(audio, sample_rate) + outputs = session.run(None, {'input_features': input_features}) + probability = float(outputs[0].flatten()[0]) + prediction = 'complete' if probability > 0.5 else 'incomplete' + timing_ms = (time.perf_counter() - start) * 1000 + return StageResult(stage='acoustic.smart_turn_onnx', timing_ms=timing_ms, output={'decision': prediction, 'probability': probability}, available=True, provenance='real_checkpoint') \ No newline at end of file diff --git a/backend/streaming_semantic.py b/backend/streaming_semantic.py new file mode 100644 index 0000000000000000000000000000000000000000..0b419767c1ef4e8b2e41dfaa46df32ec66e7ef78 --- /dev/null +++ b/backend/streaming_semantic.py @@ -0,0 +1,59 @@ +from __future__ import annotations +import math +import time +import numpy as np +import torch +import config +from backend.semantic import classify_transcript +from backend.types import StageResult +_device = 'mps' if torch.backends.mps.is_available() else 'cpu' +_ctc_model = None +_ctc_processor = None +DEFAULT_CHUNK_MS = 1000 +MAX_STEPS = 10 + +def _get_ctc(): + global _ctc_model, _ctc_processor + if _ctc_model is None: + from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor + _ctc_processor = Wav2Vec2Processor.from_pretrained(config.HINDI_CTC_ID) + _ctc_model = Wav2Vec2ForCTC.from_pretrained(config.HINDI_CTC_ID).to(_device).eval() + return (_ctc_processor, _ctc_model) + +@torch.inference_mode() +def transcribe_ctc_chunk(audio_prefix: np.ndarray, sample_rate: int=config.SAMPLE_RATE) -> str: + processor, model = _get_ctc() + inputs = processor(audio_prefix, sampling_rate=sample_rate, return_tensors='pt', padding=True) + input_values = inputs.input_values.to(_device) + logits = model(input_values).logits + predicted_ids = torch.argmax(logits, dim=-1) + text = processor.batch_decode(predicted_ids)[0] + return text.strip() + +def run(audio: np.ndarray, sample_rate: int=config.SAMPLE_RATE, chunk_ms: int=DEFAULT_CHUNK_MS, temperature: float=0.2) -> StageResult: + start = time.perf_counter() + audio = np.asarray(audio, dtype=np.float32) + total_audio_ms = len(audio) / sample_rate * 1000 + chunk_samples = int(sample_rate * chunk_ms / 1000) + history: list[dict] = [] + first_decisive_audio_ms = None + num_steps = min(MAX_STEPS, max(1, len(audio) // chunk_samples)) + stride = max(chunk_samples, math.ceil(len(audio) / num_steps)) + for step in range(1, num_steps + 1): + prefix_len = min(len(audio), step * stride) + prefix = audio[:prefix_len] + elapsed_audio_ms = prefix_len / sample_rate * 1000 + step_start = time.perf_counter() + transcript_so_far = transcribe_ctc_chunk(prefix, sample_rate) + verdict = 'incomplete' + if transcript_so_far: + verdict = classify_transcript(transcript_so_far, temperature)['verdict'] + step_ms = (time.perf_counter() - step_start) * 1000 + history.append({'elapsed_audio_ms': round(elapsed_audio_ms, 1), 'transcript_so_far': transcript_so_far, 'verdict': verdict, 'step_processing_ms': round(step_ms, 1)}) + if first_decisive_audio_ms is None and verdict != 'incomplete': + first_decisive_audio_ms = elapsed_audio_ms + if prefix_len >= len(audio): + break + final = history[-1] + timing_ms = (time.perf_counter() - start) * 1000 + return StageResult(stage='semantic.qwen_local_streaming', timing_ms=timing_ms, output={'transcript': final['transcript_so_far'], 'verdict': final['verdict'], 'history': history, 'total_audio_ms': round(total_audio_ms, 1), 'first_decisive_audio_ms': first_decisive_audio_ms, 'eou_delay_saved_ms': round(total_audio_ms - first_decisive_audio_ms, 1) if first_decisive_audio_ms is not None else None}, available=True, provenance='architecture_reimplemented') \ No newline at end of file diff --git a/backend/types.py b/backend/types.py new file mode 100644 index 0000000000000000000000000000000000000000..443df92d93344571bc2a011c21e9be57c4055c87 --- /dev/null +++ b/backend/types.py @@ -0,0 +1,65 @@ +from __future__ import annotations +from dataclasses import dataclass, field +from typing import Any, Literal, Optional +Provenance = Literal['real_checkpoint', 'trained_by_us', 'rule', 'architecture_reimplemented', 'unavailable'] +GateChoice = Literal['webrtcvad', 'silero_vad', 'none'] +EncoderChoice = Literal['whisper_tiny', 'whisper_base', 'wav2vec2', 'smart_turn_onnx', 'none'] +PoolingChoice = Literal['mean', 'cross_attention'] +HeadChoice = Literal['linear', 'mlp'] +SemanticChoice = Literal['off', 'qwen_local', 'qwen_local_streaming', 'livekit_eou', 'groq_api', 'openrouter_api'] +FusionChoice = Literal['off', 'weighted_vote', 'easy_turn'] +OutputClasses = Literal['binary', '3class', '4class'] +FullDuplexChoice = Literal['off', 'moshi', 'human1'] + +class StageUnavailableError(RuntimeError): + pass + +@dataclass +class StageResult: + stage: str + timing_ms: float + output: Any = None + available: bool = True + reason: Optional[str] = None + provenance: Optional[Provenance] = None + +@dataclass +class PipelineConfig: + gate: GateChoice = 'silero_vad' + encoder: EncoderChoice = 'whisper_tiny' + pooling: PoolingChoice = 'mean' + head: HeadChoice = 'linear' + semantic: SemanticChoice = 'off' + fusion: FusionChoice = 'off' + output_classes: OutputClasses = 'binary' + full_duplex: FullDuplexChoice = 'off' + vad_aggressiveness: int = 2 + silence_trigger_ms: int = 400 + semantic_temperature: float = 0.2 + acoustic_weight: float = 0.6 + + def __post_init__(self) -> None: + if self.encoder == 'none': + if self.semantic == 'off': + raise ValueError("encoder='none' requires the semantic branch to be on (nothing would decide).") + if self.fusion != 'off': + raise ValueError("encoder='none' has no acoustic score to fuse with - set fusion='off'.") + if self.encoder == 'smart_turn_onnx': + if self.pooling != 'mean' or self.head != 'linear': + raise ValueError("encoder='smart_turn_onnx' is a single opaque preset and cannot be combined with a separate pooling/head choice (see docs/decision-log.md #18).") + if self.fusion != 'off' and self.semantic == 'off': + raise ValueError('fusion requires the semantic branch to be on.') + if self.full_duplex != 'off': + pass + +@dataclass +class PipelineResult: + config: PipelineConfig + decision: Optional[str] = None + probability: Optional[float] = None + transcript: Optional[str] = None + semantic_verdict: Optional[str] = None + stage_results: list[StageResult] = field(default_factory=list) + total_latency_ms: float = 0.0 + provenance: Optional[Provenance] = None + preset_label: Optional[str] = None \ No newline at end of file diff --git a/config.py b/config.py new file mode 100644 index 0000000000000000000000000000000000000000..f4bc1d0a97c63751016d273882b852bcd50d207a --- /dev/null +++ b/config.py @@ -0,0 +1,33 @@ +from pathlib import Path +ROOT = Path(__file__).resolve().parent +DATABASE_DIR = ROOT / 'database' +AUDIO_DIR = DATABASE_DIR / 'audio' +SAMPLE_CLIPS_DIR = AUDIO_DIR / 'sample_clips' +EVAL_DIR = AUDIO_DIR / 'eval' +OOD_TEST_CLIPS_DIR = EVAL_DIR / 'ood_test_clips' +TRUNCATION_PILOT_DIR = EVAL_DIR / 'truncation_pilot' +LIVE_RUNS_DIR = AUDIO_DIR / 'live_runs' +CHECKPOINTS_DIR = DATABASE_DIR / 'checkpoints' +MODELS_DIR = DATABASE_DIR / 'models' +EASY_TURN_DIR = MODELS_DIR / 'easy_turn' +CACHE_DIR = DATABASE_DIR / 'cache' +DATA_CACHE_DIR = CACHE_DIR / 'dataset' +OOD_CACHE_DIR = CACHE_DIR / 'ood_test' +EXPERIMENTS_DIR = ROOT / 'experiments' +EMBEDDINGS_CACHE_DIR = EXPERIMENTS_DIR / 'embeddings_cache' +RESULTS_DIR = EXPERIMENTS_DIR / 'results' +SAMPLE_RATE = 16000 +SMART_TURN_WINDOW_SECONDS = 8 +SMART_TURN_WINDOW_SAMPLES = SMART_TURN_WINDOW_SECONDS * SAMPLE_RATE +WHISPER_TINY_ID = 'openai/whisper-tiny' +WHISPER_BASE_ID = 'openai/whisper-base' +WAV2VEC2_ID = 'facebook/wav2vec2-base' +QWEN_LOCAL_ID = 'Qwen/Qwen2.5-0.5B-Instruct' +HINDI_CTC_ID = 'theainerd/Wav2Vec2-large-xlsr-hindi' +LIVEKIT_TURN_DETECTOR_ID = 'livekit/turn-detector' +SMART_TURN_REPO = 'pipecat-ai/smart-turn-v3' +SMART_TURN_ONNX_FILENAME = 'smart-turn-v3.2-cpu.onnx' +DATASET_REPO = 'pipecat-ai/smart-turn-data-v3.2-train' +DATASET_NUM_SHARDS_TOTAL = 83 +EASY_TURN_REPO = 'ASLP-lab/Easy-Turn' +ENCODER_IDS = {'whisper_tiny': WHISPER_TINY_ID, 'whisper_base': WHISPER_BASE_ID, 'wav2vec2': WAV2VEC2_ID} \ No newline at end of file diff --git a/database/audio/sample_clips/english_fleurs_1511.wav b/database/audio/sample_clips/english_fleurs_1511.wav new file mode 100644 index 0000000000000000000000000000000000000000..81857e3a83b2213db1f4799a0cb7e38ad97fb5ad --- /dev/null +++ b/database/audio/sample_clips/english_fleurs_1511.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9dde919c8a85ece3116cf42fa3f2d052a9174319a4062c975631c848594961d +size 180524 diff --git a/database/audio/sample_clips/english_fleurs_1512.wav b/database/audio/sample_clips/english_fleurs_1512.wav new file mode 100644 index 0000000000000000000000000000000000000000..86652035ebaf95f423ff02698506c677c54acd84 --- /dev/null +++ b/database/audio/sample_clips/english_fleurs_1512.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c30bddf019ff7abf8f81f0d6a0fc018b186613a9b2d67c3f3df24ae255e0b6b1 +size 345644 diff --git a/database/audio/sample_clips/english_fleurs_1545.wav b/database/audio/sample_clips/english_fleurs_1545.wav new file mode 100644 index 0000000000000000000000000000000000000000..502ccf100d882e018b508074ac22b46352e8fd6a --- /dev/null +++ b/database/audio/sample_clips/english_fleurs_1545.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee23300897063f00999fb4a5f3bfd7521152b596ad91c6bf4758be54a855c2ba +size 245164 diff --git a/database/audio/sample_clips/english_fleurs_1546.wav b/database/audio/sample_clips/english_fleurs_1546.wav new file mode 100644 index 0000000000000000000000000000000000000000..d65ec40e621d45591af133a932c9563c33700392 --- /dev/null +++ b/database/audio/sample_clips/english_fleurs_1546.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be0fb2731a5012de1902223605b2a5110828482b65aa48b869eebaf319d2cdaf +size 391724 diff --git a/database/audio/sample_clips/english_fleurs_1554.wav b/database/audio/sample_clips/english_fleurs_1554.wav new file mode 100644 index 0000000000000000000000000000000000000000..9c2dfe7944e9c5adbf2b8900278b4c990946f917 Binary files /dev/null and b/database/audio/sample_clips/english_fleurs_1554.wav differ diff --git a/database/audio/sample_clips/english_fleurs_1578.wav b/database/audio/sample_clips/english_fleurs_1578.wav new file mode 100644 index 0000000000000000000000000000000000000000..aad7dc11e41219754278d89c7112b09a1ce97edc --- /dev/null +++ b/database/audio/sample_clips/english_fleurs_1578.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:933a4569cc229f58e56cf680fa4cd3c7660b1e73cf6ef734ed906699bb81b1f9 +size 265644 diff --git a/database/audio/sample_clips/english_fleurs_1595.wav b/database/audio/sample_clips/english_fleurs_1595.wav new file mode 100644 index 0000000000000000000000000000000000000000..d400041e8ce32d19f63f918fbd05cb439327e07c --- /dev/null +++ b/database/audio/sample_clips/english_fleurs_1595.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d736aa4815a6de4ecc1f2864c4e6f9c67906b8eb37e8df103a4107d95d257377 +size 320684 diff --git a/database/audio/sample_clips/english_fleurs_1637.wav b/database/audio/sample_clips/english_fleurs_1637.wav new file mode 100644 index 0000000000000000000000000000000000000000..314f35ddbf1ec49da6d933e447452823453b2edc --- /dev/null +++ b/database/audio/sample_clips/english_fleurs_1637.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bad0d5ef9019d1c9cbbc4aac7abe0ea49f5b4fd7956f92a369f70bc4946ae5f +size 1015724 diff --git a/database/audio/sample_clips/english_fleurs_1639.wav b/database/audio/sample_clips/english_fleurs_1639.wav new file mode 100644 index 0000000000000000000000000000000000000000..84e4ba6af453a014941653a0f97baf72a3b00a43 --- /dev/null +++ b/database/audio/sample_clips/english_fleurs_1639.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78af7de3a38af8b7a368096b7047650fc387491601bfe48221806cf46d94ec92 +size 291884 diff --git a/database/audio/sample_clips/english_fleurs_1645.wav b/database/audio/sample_clips/english_fleurs_1645.wav new file mode 100644 index 0000000000000000000000000000000000000000..f028f2ff91998efbf8c33727a539c89acfe5afc4 --- /dev/null +++ b/database/audio/sample_clips/english_fleurs_1645.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15a992f941349e2a03fafca3d96ea1a2d86ffb6aadf8833369095b21423e6042 +size 218924 diff --git a/database/audio/sample_clips/hindi_fleurs_1524.wav b/database/audio/sample_clips/hindi_fleurs_1524.wav new file mode 100644 index 0000000000000000000000000000000000000000..230b92d068afe2ed47a6e859f6997908db821fbc --- /dev/null +++ b/database/audio/sample_clips/hindi_fleurs_1524.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e00d95d250c2ba7a3f2186b26f82a126e80b61bb667bd0ee2af712287c620bef +size 382124 diff --git a/database/audio/sample_clips/hindi_fleurs_1526.wav b/database/audio/sample_clips/hindi_fleurs_1526.wav new file mode 100644 index 0000000000000000000000000000000000000000..03eb63910cbc55b9a9f7841e875c44fa4d031f08 --- /dev/null +++ b/database/audio/sample_clips/hindi_fleurs_1526.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1b686404e5dadd0cdadc91c20737c9d5846b0ceb981f533576f1c1ec1c49cd1 +size 453164 diff --git a/database/audio/sample_clips/hindi_fleurs_1540.wav b/database/audio/sample_clips/hindi_fleurs_1540.wav new file mode 100644 index 0000000000000000000000000000000000000000..f145023e3b8f4fcc69c6102f71c3bd184cea4c0c --- /dev/null +++ b/database/audio/sample_clips/hindi_fleurs_1540.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2053febb046fab7e62066d77ecefa597f80db86e02843517400d4192d2df270 +size 276524 diff --git a/database/audio/sample_clips/hindi_fleurs_1549.wav b/database/audio/sample_clips/hindi_fleurs_1549.wav new file mode 100644 index 0000000000000000000000000000000000000000..5587367ae3666c391c41c107b10e354dd68d90f2 --- /dev/null +++ b/database/audio/sample_clips/hindi_fleurs_1549.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:855c89aa564662385d32b05d35e28edc60a6d05974735320886f913c87c493ee +size 506924 diff --git a/database/audio/sample_clips/hindi_fleurs_1560.wav b/database/audio/sample_clips/hindi_fleurs_1560.wav new file mode 100644 index 0000000000000000000000000000000000000000..0722ea4cf9f3225f6ed80a7206858088cc8f41c3 --- /dev/null +++ b/database/audio/sample_clips/hindi_fleurs_1560.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c24145d731abae2b8e1323ff08edd944f8a2a1ed05035ebf43770f8190f8f9f +size 192044 diff --git a/database/audio/sample_clips/hindi_fleurs_1581.wav b/database/audio/sample_clips/hindi_fleurs_1581.wav new file mode 100644 index 0000000000000000000000000000000000000000..b0ce9466bc3c50263050f1e797ddea9abdfaf6f6 --- /dev/null +++ b/database/audio/sample_clips/hindi_fleurs_1581.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c58925c65f89f1ddeba6055d59b27e37ae49bf1ca0233ae0a1d1747f60a3168 +size 224684 diff --git a/database/audio/sample_clips/hindi_fleurs_1609.wav b/database/audio/sample_clips/hindi_fleurs_1609.wav new file mode 100644 index 0000000000000000000000000000000000000000..c961dbbf1e67e05f1e07056e808b0775c5282735 --- /dev/null +++ b/database/audio/sample_clips/hindi_fleurs_1609.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4871a032b62c48f1d286d0b977b8218856c1205360359f96a273304870007e7c +size 303404 diff --git a/database/audio/sample_clips/hindi_fleurs_1620.wav b/database/audio/sample_clips/hindi_fleurs_1620.wav new file mode 100644 index 0000000000000000000000000000000000000000..74fe4e5e9cfeb8cfd114e9a9d6c0eb56e61697a0 --- /dev/null +++ b/database/audio/sample_clips/hindi_fleurs_1620.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41801b96eb739e2f6a0850c7f9cc75adb36468a9a81613922996276ac4bd4f38 +size 610604 diff --git a/database/audio/sample_clips/hindi_fleurs_1641.wav b/database/audio/sample_clips/hindi_fleurs_1641.wav new file mode 100644 index 0000000000000000000000000000000000000000..6d414a861caebdc089891f33296748288e594e59 --- /dev/null +++ b/database/audio/sample_clips/hindi_fleurs_1641.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c81049dd7cb390aa3267410ac4e1e2e95622d6085366fe3ff088471324cd8a5 +size 361004 diff --git a/database/audio/sample_clips/hindi_fleurs_1645.wav b/database/audio/sample_clips/hindi_fleurs_1645.wav new file mode 100644 index 0000000000000000000000000000000000000000..034fd882d4ea3aa2bbfc53e28d758df203aa3071 --- /dev/null +++ b/database/audio/sample_clips/hindi_fleurs_1645.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f1aee1a6596889197682524a5ad105141d38b54318ed03e1bf3580b64f36c39 +size 169004 diff --git a/database/audio/sample_clips/hinglish_hiacc_AD09072.wav b/database/audio/sample_clips/hinglish_hiacc_AD09072.wav new file mode 100644 index 0000000000000000000000000000000000000000..661ae1d9080066cf97c1fb31bf5d5c499c7eadb7 --- /dev/null +++ b/database/audio/sample_clips/hinglish_hiacc_AD09072.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd452e54e2e1432586f68f05d2cd25d0411d83dafc8f4dac000ff4cb59bc9597 +size 512084 diff --git a/database/audio/sample_clips/hinglish_hiacc_AD13003.wav b/database/audio/sample_clips/hinglish_hiacc_AD13003.wav new file mode 100644 index 0000000000000000000000000000000000000000..a3d44dab4c4000fe525184cd2efabca7cda12d73 Binary files /dev/null and b/database/audio/sample_clips/hinglish_hiacc_AD13003.wav differ diff --git a/database/audio/sample_clips/hinglish_hiacc_AD22092.wav b/database/audio/sample_clips/hinglish_hiacc_AD22092.wav new file mode 100644 index 0000000000000000000000000000000000000000..a9075e159e516c6dd99c87700ae4ae91b4c24413 --- /dev/null +++ b/database/audio/sample_clips/hinglish_hiacc_AD22092.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77a8459e9696d11be084b1671b8bca53d0c5275f52f4eb2a1c834d94992a94fd +size 227710 diff --git a/database/audio/sample_clips/hinglish_hiacc_AD23025.wav b/database/audio/sample_clips/hinglish_hiacc_AD23025.wav new file mode 100644 index 0000000000000000000000000000000000000000..3e5af3f1c38c5ff378c748c79aff0bdc809e3c95 --- /dev/null +++ b/database/audio/sample_clips/hinglish_hiacc_AD23025.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:599e5b5ccd3c1dbf321963d158eda124e3e6ca2c582b03e1786474732da7ddcb +size 104730 diff --git a/database/audio/sample_clips/hinglish_hiacc_AD36007.wav b/database/audio/sample_clips/hinglish_hiacc_AD36007.wav new file mode 100644 index 0000000000000000000000000000000000000000..fa940d8e18dc02fa68424ef0528732611b715518 --- /dev/null +++ b/database/audio/sample_clips/hinglish_hiacc_AD36007.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8223b79e5de802db3adb45d4bf724c9eadaf79701d20c357b1a56ea189dccc0 +size 137886 diff --git a/database/audio/sample_clips/hinglish_hiacc_AD36064.wav b/database/audio/sample_clips/hinglish_hiacc_AD36064.wav new file mode 100644 index 0000000000000000000000000000000000000000..87070fd5be77712439ddbc63c344c6e54a475f1e --- /dev/null +++ b/database/audio/sample_clips/hinglish_hiacc_AD36064.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f90d8b109474071f39fd8767c50d4afb5d7b22dd83e91f0ece9c3edb707426d7 +size 121006 diff --git a/database/audio/sample_clips/hinglish_hiacc_AD40049.wav b/database/audio/sample_clips/hinglish_hiacc_AD40049.wav new file mode 100644 index 0000000000000000000000000000000000000000..4c547bf5502b0efa535099ad2e5d210781653d04 --- /dev/null +++ b/database/audio/sample_clips/hinglish_hiacc_AD40049.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5234def68e6e0823873e2dc63f2586a3986f2ddf4ecb53a2d6d31945198ee34e +size 289802 diff --git a/database/audio/sample_clips/hinglish_hiacc_AD40103.wav b/database/audio/sample_clips/hinglish_hiacc_AD40103.wav new file mode 100644 index 0000000000000000000000000000000000000000..00882b8e43ccf9ecaf3c3a3bb51aa9a4190efa21 --- /dev/null +++ b/database/audio/sample_clips/hinglish_hiacc_AD40103.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dc4b1146f75f1540b9348c02b4277d205189c41ebf1a65c8ba3ce939fefa607 +size 193346 diff --git a/database/audio/sample_clips/hinglish_hiacc_AD40158.wav b/database/audio/sample_clips/hinglish_hiacc_AD40158.wav new file mode 100644 index 0000000000000000000000000000000000000000..459e3d2dfc1f2b97957651f01dcaa23bf1ab339b --- /dev/null +++ b/database/audio/sample_clips/hinglish_hiacc_AD40158.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96eeca3c3bd867f0b4f953843004127cc1d6f486dcc6d637a6740d4cdd29a0a9 +size 173452 diff --git a/database/audio/sample_clips/hinglish_hiacc_AD60096.wav b/database/audio/sample_clips/hinglish_hiacc_AD60096.wav new file mode 100644 index 0000000000000000000000000000000000000000..73fd2ff8565bb839ef753a9ba2cfca84d6dd6d53 --- /dev/null +++ b/database/audio/sample_clips/hinglish_hiacc_AD60096.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:778f8fe8a0fa53585b93e61d94270f34cc681297ca5da4d2f0c9fec53d8a6a3f +size 153124 diff --git a/database/audio/sample_clips/manifest.json b/database/audio/sample_clips/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..bd18882f1d93cdf1be4a3e1f62435b94b25e4519 --- /dev/null +++ b/database/audio/sample_clips/manifest.json @@ -0,0 +1,232 @@ +[ + { + "filename": "english_fleurs_1554.wav", + "bucket": "english", + "source": "fleurs_en_us", + "duration_seconds": 2.7, + "transcript": "he built a wifi door bell he said" + }, + { + "filename": "english_fleurs_1511.wav", + "bucket": "english", + "source": "fleurs_en_us", + "duration_seconds": 5.64, + "transcript": "soon officers equipped with riot gear entered the yard and cornered the inmates with tear gas" + }, + { + "filename": "english_fleurs_1645.wav", + "bucket": "english", + "source": "fleurs_en_us", + "duration_seconds": 6.84, + "transcript": "its all-pervading power affected everyone from king to commoner" + }, + { + "filename": "english_fleurs_1545.wav", + "bucket": "english", + "source": "fleurs_en_us", + "duration_seconds": 7.66, + "transcript": "a curry can be either dry or wet depending on the amount of liquid" + }, + { + "filename": "english_fleurs_1578.wav", + "bucket": "english", + "source": "fleurs_en_us", + "duration_seconds": 8.3, + "transcript": "then lakkha singh took the lead in singing the bhajans" + }, + { + "filename": "english_fleurs_1639.wav", + "bucket": "english", + "source": "fleurs_en_us", + "duration_seconds": 9.12, + "transcript": "since students are often the most critical audience the blog writer begins to strive to improve writing to avoid criticism" + }, + { + "filename": "english_fleurs_1595.wav", + "bucket": "english", + "source": "fleurs_en_us", + "duration_seconds": 10.02, + "transcript": "the tibetan buddhism is based on the teachings of buddha but were extended by the mahayana path of love and by a lot of techniques from indian yoga" + }, + { + "filename": "english_fleurs_1512.wav", + "bucket": "english", + "source": "fleurs_en_us", + "duration_seconds": 10.8, + "transcript": "this new environment has different resources and different competitors so the new population will need different features or adaptations to be a strong competitor than what they had needed before" + }, + { + "filename": "english_fleurs_1546.wav", + "bucket": "english", + "source": "fleurs_en_us", + "duration_seconds": 12.24, + "transcript": "the feathers' structure suggests that they were not used in flight but rather for temperature regulation or display the researchers suggested that even though this is the tail of a young dinosaur the sample shows adult plumage and not a chick's down" + }, + { + "filename": "english_fleurs_1637.wav", + "bucket": "english", + "source": "fleurs_en_us", + "duration_seconds": 31.74, + "transcript": "the terrified king louis xvi queen marie antoinette their two young children 11 year old marie therese and four year old louis-charles and the king's sister madam elizabeth on the 6th october 1789 were forced back to paris from versailles by a mob of market women" + }, + { + "filename": "hindi_fleurs_1645.wav", + "bucket": "hindi", + "source": "fleurs_hi_in", + "duration_seconds": 5.28, + "transcript": "\u0907\u0938\u0915\u0940 \u0938\u0930\u094d\u0935\u0935\u094d\u092f\u093e\u092a\u0940 \u0936\u0915\u094d\u0924\u093f \u0928\u0947 \u0930\u093e\u091c\u093e \u0938\u0947 \u0932\u0947\u0915\u0930 \u0906\u092e \u0924\u0915 \u0938\u092d\u0940 \u0915\u094b \u092a\u094d\u0930\u092d\u093e\u0935\u093f\u0924 \u0915\u093f\u092f\u093e" + }, + { + "filename": "hindi_fleurs_1560.wav", + "bucket": "hindi", + "source": "fleurs_hi_in", + "duration_seconds": 6.0, + "transcript": "\u091c\u0948\u0938\u0947 \u0939\u0940 \u0906\u092a \u0927\u093e\u0930\u093e \u0938\u0947 \u0928\u093f\u0915\u0932 \u0906\u0924\u0947 \u0939\u0948\u0902 \u092a\u0940\u091b\u0947 \u0915\u0940 \u0913\u0930 \u0924\u0948\u0930\u0928\u0947 \u092e\u0947\u0902 \u0915\u094b\u0908 \u0916\u093e\u0938 \u092e\u0941\u0936\u094d\u0915\u093f\u0932 \u0928\u0939\u0940\u0902 \u0939\u094b\u0924\u0940 \u0939\u0948" + }, + { + "filename": "hindi_fleurs_1581.wav", + "bucket": "hindi", + "source": "fleurs_hi_in", + "duration_seconds": 7.02, + "transcript": "\u0905\u092e\u0947\u091c\u093c\u0928 \u0927\u0930\u0924\u0940 \u0915\u0940 \u0938\u092c\u0938\u0947 \u091a\u094c\u0921\u093c\u0940 \u0928\u0926\u0940 \u092d\u0940 \u0939\u0948 \u091c\u094b \u0915\u092d\u0940-\u0915\u092d\u0940 \u091b\u0939 \u092e\u0940\u0932 \u091c\u093f\u0924\u0928\u0940 \u091a\u094c\u0921\u093c\u0940 \u0939\u094b \u091c\u093e\u0924\u0940 \u0939\u0948" + }, + { + "filename": "hindi_fleurs_1540.wav", + "bucket": "hindi", + "source": "fleurs_hi_in", + "duration_seconds": 8.64, + "transcript": "\u0938\u0948\u0926\u094d\u0927\u093e\u0902\u0924\u093f\u0915 \u0930\u0942\u092a \u0938\u0947 \u0924\u093f\u092c\u094d\u092c\u0924\u0940 \u092c\u094c\u0926\u094d\u0927 \u0927\u0930\u094d\u092e \u092c\u0939\u0941\u0924 \u0938\u0930\u0932 \u0939\u0948 \u0907\u0938\u092e\u0947\u0902 \u0915\u0941\u0902\u0921\u0932\u093f\u0928\u0940 \u092f\u094b\u0917 \u0927\u094d\u092f\u093e\u0928 \u0914\u0930 \u0938\u092d\u0940 \u0915\u094b \u092a\u094d\u0930\u0947\u092e \u0938\u0947 \u0917\u0932\u0947 \u0932\u0917\u093e\u0928\u0947 \u0935\u093e\u0932\u093e \u092e\u093e\u0930\u094d\u0917 \u0936\u093e\u092e\u093f\u0932 \u0939\u0948" + }, + { + "filename": "hindi_fleurs_1609.wav", + "bucket": "hindi", + "source": "fleurs_hi_in", + "duration_seconds": 9.48, + "transcript": "\u0930\u0949\u092c\u093f\u0928 \u0909\u0925\u092a\u094d\u092a\u093e \u0928\u0947 \u0915\u0947\u0935\u0932 41 \u0917\u0947\u0902\u0926\u094b\u0902 \u092e\u0947\u0902 11 \u091a\u094c\u0915\u094b\u0902 \u0914\u0930 2 \u091b\u0915\u094d\u0915\u094b\u0902 \u0915\u0940 \u092e\u0926\u0926 \u0938\u0947 70 \u0930\u0928 \u092c\u0928\u093e \u0915\u0930 \u092a\u093e\u0930\u0940 \u0915\u093e \u0938\u0930\u094d\u0935\u094b\u091a\u094d\u091a \u0938\u094d\u0915\u094b\u0930 \u092c\u0928\u093e\u092f\u093e" + }, + { + "filename": "hindi_fleurs_1641.wav", + "bucket": "hindi", + "source": "fleurs_hi_in", + "duration_seconds": 11.28, + "transcript": "\u0939\u093e\u0932\u093e\u0902\u0915\u093f \u091c\u0902\u0917\u0932 \u0915\u093e \u0905\u0930\u094d\u0925 \u0915\u0947\u0935\u0932 \u092e\u0948\u0902\u0917\u094d\u0930\u094b\u0935 \u0926\u0932\u0926\u0932 \u0928\u0939\u0940\u0902 \u0939\u0948\u0902 \u0935\u0947 \u0915\u0941\u091b \u0905\u0902\u0924\u093f\u092e \u0936\u0915\u094d\u0924\u093f\u0936\u093e\u0932\u0940 \u091c\u0902\u0917\u0932\u094b\u0902 \u092e\u0947\u0902 \u0938\u0947 \u0939\u0948\u0902 \u091c\u094b \u0915\u092d\u0940 \u0917\u0902\u0917\u093e \u092a\u0920\u093e\u0930 \u0915\u094b \u0915\u0935\u0930 \u0915\u0930\u0924\u0947 \u0925\u0947" + }, + { + "filename": "hindi_fleurs_1524.wav", + "bucket": "hindi", + "source": "fleurs_hi_in", + "duration_seconds": 11.94, + "transcript": "\u092e\u0940\u091f\u094d\u0930\u093f\u0915 \u092a\u094d\u0930\u0923\u093e\u0932\u0940 \u0915\u0947 \u0907\u0938\u094d\u0924\u0947\u092e\u093e\u0932 \u0928\u093f\u0930\u092a\u0947\u0915\u094d\u0937\u0935\u093e\u0926 \u0938\u0947 \u0917\u0923\u0924\u0902\u0924\u094d\u0930\u0935\u093e\u0926 \u0930\u093e\u0937\u094d\u091f\u094d\u0930\u0935\u093e\u0926 \u092e\u0947\u0902 \u092c\u0926\u0932\u093e\u0935 \u0914\u0930 \u0926\u0947\u0936 \u0915\u093e \u092d\u0930\u094b\u0938\u093e \u0932\u094b\u0917\u094b\u0902 \u0938\u0947 \u0939\u0948 \u0915\u093f\u0938\u0940 \u090f\u0915 \u0936\u093e\u0938\u0915 \u0938\u0947 \u0928\u0939\u0940\u0902 \u0910\u0938\u0940 \u092c\u093e\u0924\u094b\u0902 \u0915\u093e \u092c\u0939\u0941\u0924 \u091c\u093c\u094d\u092f\u093e\u0926\u093e \u0938\u093e\u092e\u093e\u091c\u093f\u0915 \u0914\u0930 \u0930\u093e\u091c\u0928\u0948\u0924\u093f\u0915 \u0905\u0938\u0930 \u0939\u094b\u0924\u093e \u0939\u0948" + }, + { + "filename": "hindi_fleurs_1526.wav", + "bucket": "hindi", + "source": "fleurs_hi_in", + "duration_seconds": 14.16, + "transcript": "\u092f\u0926\u093f \u0906\u092a \u0938\u0930\u094d\u0926\u093f\u092f\u094b\u0902 \u092e\u0947\u0902 \u0906\u0930\u094d\u0915\u091f\u093f\u0915 \u092f\u093e \u0905\u0902\u091f\u093e\u0930\u094d\u0915\u091f\u093f\u0915 \u0915\u094d\u0937\u0947\u0924\u094d\u0930\u094b\u0902 \u092e\u0947\u0902 \u091c\u093e\u0924\u0947 \u0939\u0948\u0902 \u0924\u094b \u0906\u092a \u0927\u094d\u0930\u0941\u0935\u0940\u092f \u0930\u093e\u0924 \u0915\u093e \u0905\u0928\u0941\u092d\u0935 \u0915\u0930\u0947\u0902\u0917\u0947 \u091c\u093f\u0938\u0915\u093e \u0905\u0930\u094d\u0925 \u0939\u0948 \u0915\u093f \u0938\u0942\u0930\u091c \u0915\u094d\u0937\u093f\u0924\u093f\u091c \u0938\u0947 \u090a\u092a\u0930 \u0928\u0939\u0940\u0902 \u0909\u0917\u0924\u093e \u0939\u0948" + }, + { + "filename": "hindi_fleurs_1549.wav", + "bucket": "hindi", + "source": "fleurs_hi_in", + "duration_seconds": 15.84, + "transcript": "\u0907\u0928 \u0938\u093f\u0926\u094d\u0927\u093e\u0902\u0924\u094b\u0902 \u0938\u0947 \u0938\u0902\u0915\u0947\u0924 \u092e\u093f\u0932\u0924\u093e \u0939\u0948 \u0915\u093f \u0932\u094b\u0917\u094b\u0902 \u0915\u0940 \u0915\u0941\u091b \u0910\u0938\u0940 \u091c\u093c\u0930\u0942\u0930\u0924\u0947\u0902 \u0914\u0930/\u092f\u093e \u0907\u091a\u094d\u091b\u093e\u090f\u0901 \u0925\u0940\u0902 \u091c\u094b \u0909\u0928\u0915\u0947 \u0935\u092f\u0938\u094d\u0915 \u0939\u094b\u0924\u0947-\u0939\u094b\u0924\u0947 \u0909\u0928\u0915\u0947 \u092d\u0940\u0924\u0930 \u0928\u093f\u0939\u093f\u0924 \u0939\u094b \u0917\u092f\u0940" + }, + { + "filename": "hindi_fleurs_1620.wav", + "bucket": "hindi", + "source": "fleurs_hi_in", + "duration_seconds": 19.08, + "transcript": "\u091c\u093e\u0935\u093e \u0915\u0947 \u0935\u094d\u092f\u0902\u091c\u0928 \u0905\u092c \u092a\u0942\u0930\u0947 \u0926\u094d\u0935\u0940\u092a\u0938\u092e\u0942\u0939 \u092e\u0947\u0902 \u0935\u094d\u092f\u093e\u092a\u0915 \u0930\u0942\u092a \u0938\u0947 \u0909\u092a\u0932\u092c\u094d\u0927 \u0939\u0948\u0902 \u091c\u093f\u0938\u092e\u0947\u0902 \u092a\u093e\u0930\u0902\u092a\u0930\u093f\u0915 \u0935\u094d\u092f\u0902\u091c\u0928\u094b\u0902 \u0915\u0940 \u090f\u0915 \u0935\u093f\u0936\u0947\u0937\u0924\u093e \u0939\u0948 \u091c\u093e\u0935\u093e \u0915\u0947 \u0935\u094d\u092f\u0902\u091c\u0928\u094b\u0902 \u092e\u0947\u0902 \u092a\u094d\u0930\u092e\u0941\u0916 \u0938\u094d\u0935\u093e\u0926 \u092e\u0942\u0902\u0917\u092b\u0932\u0940 \u092e\u093f\u0930\u094d\u091a \u091a\u0940\u0928\u0940 \u0935\u093f\u0936\u0947\u0937 \u0930\u0942\u092a \u0938\u0947 \u091c\u093e\u0935\u093e \u0915\u0940 \u0928\u093e\u0930\u093f\u092f\u0932 \u091a\u0940\u0928\u0940 \u0914\u0930 \u0935\u093f\u092d\u093f\u0928\u094d\u0928 \u0938\u0941\u0917\u0902\u0927\u093f\u0924 \u092e\u0938\u093e\u0932\u094b\u0902 \u0915\u093e \u0939\u094b\u0924\u093e \u0939\u0948\u0964" + }, + { + "filename": "hinglish_hiacc_AD13003.wav", + "bucket": "hinglish", + "source": "hiacc_adult_test", + "duration_seconds": 3.02, + "transcript": "So \u092e\u0947\u0930\u093e favourite festival \u0939\u0948 Diwali", + "code_mixing_index": 33.33, + "code_switch_count": 4 + }, + { + "filename": "hinglish_hiacc_AD23025.wav", + "bucket": "hinglish", + "source": "hiacc_adult_test", + "duration_seconds": 3.27, + "transcript": "\u0914\u0930 \u0909\u0928\u0915\u0940 friendship \u092c\u093f\u0932\u094d\u0915\u0941\u0932 \u091a\u0941\u091f\u0915\u093f\u092f\u094b\u0902 \u0915\u0940 \u0924\u0930\u0939 \u0925\u0940", + "code_mixing_index": 12.5, + "code_switch_count": 2 + }, + { + "filename": "hinglish_hiacc_AD36064.wav", + "bucket": "hinglish", + "source": "hiacc_adult_test", + "duration_seconds": 3.78, + "transcript": "\u0935\u094b as a memorial \u092c\u0928\u093e \u0939\u0948 those people who lost", + "code_mixing_index": 30.0, + "code_switch_count": 3 + }, + { + "filename": "hinglish_hiacc_AD36007.wav", + "bucket": "hinglish", + "source": "hiacc_adult_test", + "duration_seconds": 4.31, + "transcript": "Major reason is because \u092e\u0941\u091d\u0947 lights \u092c\u0939\u0941\u0924 \u092a\u0938\u0902\u0926 \u0939\u0948 I love all these \u0926\u093f\u092f\u093e", + "code_mixing_index": 35.71, + "code_switch_count": 5 + }, + { + "filename": "hinglish_hiacc_AD60096.wav", + "bucket": "hinglish", + "source": "hiacc_adult_test", + "duration_seconds": 4.78, + "transcript": "\u0907\u0938\u0947 \u0938\u0941\u0928\u0915\u0947 \u0926\u094b\u0928\u094b\u0902 \u0928\u0947 \u0938\u094b\u091a\u093e \u0915\u093f \u092f\u0947 \u0909\u0928\u0915\u093e next adventure spot \u0939\u094b \u0938\u0915\u0924\u093e \u0939\u0948\u0964", + "code_mixing_index": 21.43, + "code_switch_count": 2 + }, + { + "filename": "hinglish_hiacc_AD40158.wav", + "bucket": "hinglish", + "source": "hiacc_adult_test", + "duration_seconds": 5.42, + "transcript": "\u0910\u0938\u093e \u0932\u0917 \u0930\u0939\u093e \u0939\u0948 \u091c\u0948\u0938\u0947 \u0915\u0940 superman \u092c\u0928\u0928\u0947 \u0915\u0940 \u0915\u094b\u0936\u093f\u0936 \u0915\u0930 \u0930\u0939\u093e \u0939\u0948 \u0924\u094b \u0907\u0938 image \u0938\u0947 \u092f\u0947 \u092d\u0940 \u0915\u0939\u093e \u091c\u093e \u0938\u0915\u0924\u093e \u0939\u0948 \u0915\u093f", + "code_mixing_index": 8.33, + "code_switch_count": 4 + }, + { + "filename": "hinglish_hiacc_AD40103.wav", + "bucket": "hinglish", + "source": "hiacc_adult_test", + "duration_seconds": 6.04, + "transcript": "\u091a\u093f\u0902\u091f\u0942 \u0914\u0930 \u0930\u093e\u091c\u0942 \u0928\u0947 guardian \u0915\u0947 \u0926\u093f\u090f \u0917\u090f task complete \u0915\u0930 \u0932\u093f\u090f \u0914\u0930 \u0939\u0930 \u090f\u0915 task \u0915\u0947 \u092c\u093e\u0926", + "code_mixing_index": 22.22, + "code_switch_count": 6 + }, + { + "filename": "hinglish_hiacc_AD22092.wav", + "bucket": "hinglish", + "source": "hiacc_adult_test", + "duration_seconds": 7.11, + "transcript": "\u0905\u092a\u0928\u093e \u0916\u093e\u0928\u093e \u0926\u0947\u0916 \u0915\u0947 \u0915\u094d\u092f\u0942\u0901\u0915\u093f \u090f\u0915 \u0915\u094b \u0915\u092e \u092e\u093f\u0932\u093e \u090f\u0915 \u0915\u094b \u091c\u093e\u0926\u093e \u092e\u093f\u0932\u093e \u0924\u092c \u092d\u0940 \u0935\u094b unsatisfied \u0939\u0948 \u0909\u0938\u0915\u0947 dressing sense \u0914\u0930 \u0935\u094b \u0938\u092c \u091a\u0940\u091c\u0947 \u092d\u0940 \u0905\u091a\u094d\u091b\u0940 \u0939\u0948\u0964", + "code_mixing_index": 10.71, + "code_switch_count": 4 + }, + { + "filename": "hinglish_hiacc_AD40049.wav", + "bucket": "hinglish", + "source": "hiacc_adult_test", + "duration_seconds": 9.05, + "transcript": "\u092a\u0939\u093e\u0921\u0940 \u092f\u093e \u092b\u093f\u0930 \u0915\u093f\u0938\u0940 outstation \u091c\u0948\u0938\u0947 \u0936\u0939\u0930 \u092e\u0947\u0902 \u0939\u094b \u0938\u0915\u0924\u093e \u0939\u0948 \u0915\u094d\u092f\u0942\u0915\u093f \u0935\u0939\u093e\u0901 \u092a\u0947 \u092c\u0939\u0941\u0924 different \u091c\u0917\u0939 \u0915\u0947 \u0932\u094b\u0917 \u0939\u094b\u0924\u0947 \u0939\u0948\u0902\u0964", + "code_mixing_index": 9.52, + "code_switch_count": 4 + }, + { + "filename": "hinglish_hiacc_AD09072.wav", + "bucket": "hinglish", + "source": "hiacc_adult_test", + "duration_seconds": 16.0, + "transcript": "that's a barren land Barren \u092a\u0921\u093c\u0940 \u0939\u0948 \u0910\u0938\u093e \u0932\u0917\u0924\u093e \u0939\u0948 \u0915\u093f \u092f\u0947 desert \u0939\u0948 \u090f\u0915 \u0914\u0930 there are small small Mountains are there \u091c\u094b mountains \u0939\u0948 and that are short heighted mountain \u091c\u093f\u0938\u0938\u0947 \u0915\u0940 \u0938\u093e\u092b \u0928\u091c\u0930 \u0906\u0924\u093e \u0939\u0948 \u0915\u093f \u092f\u0947 \u090f\u0915 desert \u0915\u093e part \u0939\u0948", + "code_mixing_index": 48.89, + "code_switch_count": 13 + } +] \ No newline at end of file diff --git a/database/checkpoints/whisper_base_cross_attention_linear.metadata.json b/database/checkpoints/whisper_base_cross_attention_linear.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4c11bd0b0b719b3fe2cf4e3b3f60ea622c0d63b9 --- /dev/null +++ b/database/checkpoints/whisper_base_cross_attention_linear.metadata.json @@ -0,0 +1,24 @@ +{ + "encoder": "whisper_base", + "pooling": "cross_attention", + "head": "linear", + "input_dim": 512, + "metrics": { + "train": { + "accuracy": 0.9547, + "f1": 0.953, + "n": 817 + }, + "val": { + "accuracy": 0.8851, + "f1": 0.8837, + "n": 174 + }, + "test": { + "accuracy": 0.8757, + "f1": 0.8764, + "n": 177 + } + }, + "trained_on": "smartturn (pipecat-ai/smart-turn-data-v3.2-train)" +} \ No newline at end of file diff --git a/database/checkpoints/whisper_base_cross_attention_linear.pt b/database/checkpoints/whisper_base_cross_attention_linear.pt new file mode 100644 index 0000000000000000000000000000000000000000..28a0959704c0db31684800b201f1e9cdacae3bb1 --- /dev/null +++ b/database/checkpoints/whisper_base_cross_attention_linear.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f47f8fe6484134e460125604ce308c0ea95a7156a50f81c51c9d4abd4f3c8c09 +size 6749 diff --git a/database/checkpoints/whisper_base_cross_attention_mlp.metadata.json b/database/checkpoints/whisper_base_cross_attention_mlp.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a521cdde49c3e583fe0931536cc44c181b598649 --- /dev/null +++ b/database/checkpoints/whisper_base_cross_attention_mlp.metadata.json @@ -0,0 +1,24 @@ +{ + "encoder": "whisper_base", + "pooling": "cross_attention", + "head": "mlp", + "input_dim": 512, + "metrics": { + "train": { + "accuracy": 0.9829, + "f1": 0.9817, + "n": 817 + }, + "val": { + "accuracy": 0.8621, + "f1": 0.8519, + "n": 174 + }, + "test": { + "accuracy": 0.8588, + "f1": 0.8521, + "n": 177 + } + }, + "trained_on": "smartturn (pipecat-ai/smart-turn-data-v3.2-train)" +} \ No newline at end of file diff --git a/database/checkpoints/whisper_base_cross_attention_mlp.pt b/database/checkpoints/whisper_base_cross_attention_mlp.pt new file mode 100644 index 0000000000000000000000000000000000000000..f18b171fdc318310136c2773fc511f5347976ae8 --- /dev/null +++ b/database/checkpoints/whisper_base_cross_attention_mlp.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0b0d58b293a6b03687c156a5d5b77de61b68031503e5fb73b40d1ab86c21e29 +size 268460 diff --git a/database/checkpoints/whisper_base_mean_linear.metadata.json b/database/checkpoints/whisper_base_mean_linear.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b7c502cc72e2be123b279711d2e4c1fe11c8c9f6 --- /dev/null +++ b/database/checkpoints/whisper_base_mean_linear.metadata.json @@ -0,0 +1,24 @@ +{ + "encoder": "whisper_base", + "pooling": "mean", + "head": "linear", + "input_dim": 512, + "metrics": { + "train": { + "accuracy": 0.7491, + "f1": 0.7256, + "n": 817 + }, + "val": { + "accuracy": 0.6782, + "f1": 0.6667, + "n": 174 + }, + "test": { + "accuracy": 0.7345, + "f1": 0.7006, + "n": 177 + } + }, + "trained_on": "smartturn (pipecat-ai/smart-turn-data-v3.2-train)" +} \ No newline at end of file diff --git a/database/checkpoints/whisper_base_mean_linear.pt b/database/checkpoints/whisper_base_mean_linear.pt new file mode 100644 index 0000000000000000000000000000000000000000..bdccf90e4089a357ee294b2afc80cefe3b2fbd3c --- /dev/null +++ b/database/checkpoints/whisper_base_mean_linear.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:992507748c8a7847a6ba3dc37d30bdadef3760ed9592df0c6c974bbff4748208 +size 4269 diff --git a/database/checkpoints/whisper_base_mean_mlp.metadata.json b/database/checkpoints/whisper_base_mean_mlp.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..cb2929571994b18e6d80b37de79a505bf44f2fbe --- /dev/null +++ b/database/checkpoints/whisper_base_mean_mlp.metadata.json @@ -0,0 +1,24 @@ +{ + "encoder": "whisper_base", + "pooling": "mean", + "head": "mlp", + "input_dim": 512, + "metrics": { + "train": { + "accuracy": 0.858, + "f1": 0.8509, + "n": 817 + }, + "val": { + "accuracy": 0.7471, + "f1": 0.7556, + "n": 174 + }, + "test": { + "accuracy": 0.7684, + "f1": 0.7574, + "n": 177 + } + }, + "trained_on": "smartturn (pipecat-ai/smart-turn-data-v3.2-train)" +} \ No newline at end of file diff --git a/database/checkpoints/whisper_base_mean_mlp.pt b/database/checkpoints/whisper_base_mean_mlp.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a9c47b3dc4f1c74f08345b00b797ce3783c9c8a --- /dev/null +++ b/database/checkpoints/whisper_base_mean_mlp.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ace1c8f9a1bff1fc7aedb890f43955671b739cadb752b8d5ebc7ad2dad38468 +size 265897 diff --git a/database/checkpoints/whisper_tiny_cross_attention_linear.metadata.json b/database/checkpoints/whisper_tiny_cross_attention_linear.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c8ed63798dedc6fe9b8ff5cfe6422c8aed65aa3c --- /dev/null +++ b/database/checkpoints/whisper_tiny_cross_attention_linear.metadata.json @@ -0,0 +1,24 @@ +{ + "encoder": "whisper_tiny", + "pooling": "cross_attention", + "head": "linear", + "input_dim": 384, + "metrics": { + "train": { + "accuracy": 0.9963, + "f1": 0.9963, + "n": 816 + }, + "val": { + "accuracy": 1.0, + "f1": 1.0, + "n": 174 + }, + "test": { + "accuracy": 0.9888, + "f1": 0.9888, + "n": 178 + } + }, + "trained_on": "real_hindi (HiACC + FLEURS, see experiments/results/dataset_comparison.md, docs/decision-log.md #38)" +} \ No newline at end of file diff --git a/database/checkpoints/whisper_tiny_cross_attention_linear.pt b/database/checkpoints/whisper_tiny_cross_attention_linear.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1db1786201915c0163748a6dcd60aa13d0e1f75 --- /dev/null +++ b/database/checkpoints/whisper_tiny_cross_attention_linear.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20d9b90f905e37d129b03bb5d4ff50564e3aeba859c7f4ed9fe154441e333358 +size 5725 diff --git a/database/checkpoints/whisper_tiny_cross_attention_mlp.metadata.json b/database/checkpoints/whisper_tiny_cross_attention_mlp.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2026d706d928f6760022a2cd236d378a5acc9241 --- /dev/null +++ b/database/checkpoints/whisper_tiny_cross_attention_mlp.metadata.json @@ -0,0 +1,24 @@ +{ + "encoder": "whisper_tiny", + "pooling": "cross_attention", + "head": "mlp", + "input_dim": 384, + "metrics": { + "train": { + "accuracy": 0.9461, + "f1": 0.9457, + "n": 817 + }, + "val": { + "accuracy": 0.8563, + "f1": 0.8663, + "n": 174 + }, + "test": { + "accuracy": 0.8475, + "f1": 0.8541, + "n": 177 + } + }, + "trained_on": "smartturn (pipecat-ai/smart-turn-data-v3.2-train)" +} \ No newline at end of file diff --git a/database/checkpoints/whisper_tiny_cross_attention_mlp.pt b/database/checkpoints/whisper_tiny_cross_attention_mlp.pt new file mode 100644 index 0000000000000000000000000000000000000000..def24acf5db1f59719cd5ade7fbd14511e411786 --- /dev/null +++ b/database/checkpoints/whisper_tiny_cross_attention_mlp.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d2e43eb367e6e512791a12b6a9989f99f474b1e54706158101c492f854f30e6 +size 202412 diff --git a/database/checkpoints/whisper_tiny_mean_linear.metadata.json b/database/checkpoints/whisper_tiny_mean_linear.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f5d086b63345ddcd5925d487e61cf8bdc292beef --- /dev/null +++ b/database/checkpoints/whisper_tiny_mean_linear.metadata.json @@ -0,0 +1,24 @@ +{ + "encoder": "whisper_tiny", + "pooling": "mean", + "head": "linear", + "input_dim": 384, + "metrics": { + "train": { + "accuracy": 0.6769, + "f1": 0.6185, + "n": 817 + }, + "val": { + "accuracy": 0.6322, + "f1": 0.5897, + "n": 174 + }, + "test": { + "accuracy": 0.6836, + "f1": 0.6267, + "n": 177 + } + }, + "trained_on": "smartturn (pipecat-ai/smart-turn-data-v3.2-train, the challenge dataset; its Hindi/Marathi audio is TTS-generated - see docs/decision-log.md #29, #41)" +} \ No newline at end of file diff --git a/database/checkpoints/whisper_tiny_mean_linear.pt b/database/checkpoints/whisper_tiny_mean_linear.pt new file mode 100644 index 0000000000000000000000000000000000000000..00050de37ee3135ecc36b88e6d40977d5fee3e80 --- /dev/null +++ b/database/checkpoints/whisper_tiny_mean_linear.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02437af52e31ab1846d980601d89beff6711e261f2d454857396a0aef98a583c +size 3757 diff --git a/database/checkpoints/whisper_tiny_mean_mlp.metadata.json b/database/checkpoints/whisper_tiny_mean_mlp.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e1ee43908efe460c05de7f8d11346bc53c01b75a --- /dev/null +++ b/database/checkpoints/whisper_tiny_mean_mlp.metadata.json @@ -0,0 +1,24 @@ +{ + "encoder": "whisper_tiny", + "pooling": "mean", + "head": "mlp", + "input_dim": 384, + "metrics": { + "train": { + "accuracy": 0.8176, + "f1": 0.8097, + "n": 817 + }, + "val": { + "accuracy": 0.7241, + "f1": 0.7333, + "n": 174 + }, + "test": { + "accuracy": 0.7627, + "f1": 0.747, + "n": 177 + } + }, + "trained_on": "smartturn (pipecat-ai/smart-turn-data-v3.2-train, the challenge dataset; its Hindi/Marathi audio is TTS-generated - see docs/decision-log.md #29, #41)" +} \ No newline at end of file diff --git a/database/checkpoints/whisper_tiny_mean_mlp.pt b/database/checkpoints/whisper_tiny_mean_mlp.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f3840a88ffc22fb83c72faa76427e6aa6f0e2c1 --- /dev/null +++ b/database/checkpoints/whisper_tiny_mean_mlp.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51f502a8f5399284738d13a74eca53ad018a0e7221f17bba0ba04c7b499c4e5c +size 200361 diff --git a/database/models/smart-turn-v3.2-cpu.onnx b/database/models/smart-turn-v3.2-cpu.onnx new file mode 100644 index 0000000000000000000000000000000000000000..fb8232b387c2062359083ab8d3d43bb9aaa6b0d9 --- /dev/null +++ b/database/models/smart-turn-v3.2-cpu.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bb026316b14a660486a75b1733cd3fbab8c2fd0314dc9af7be49f8cca967e4f +size 8679182 diff --git a/frontend/layout.py b/frontend/layout.py new file mode 100644 index 0000000000000000000000000000000000000000..5fcffca3e50268485cdbb54484b39e990cf06420 --- /dev/null +++ b/frontend/layout.py @@ -0,0 +1,97 @@ +from __future__ import annotations +import json +import time +import gradio as gr +import soundfile as sf +import config +from backend.presets import list_presets +from frontend import live +LOG_HEADERS = ['t_s', 'model', 'decision', 'probability', 'latency_ms'] + +def _load_sample_manifest() -> list[dict]: + manifest_path = config.SAMPLE_CLIPS_DIR / 'manifest.json' + if not manifest_path.exists(): + return [] + return json.loads(manifest_path.read_text()) +BUCKET_LABEL = {'english': 'English', 'hindi': 'Hindi', 'hinglish': 'Hinglish'} +BUCKET_ORDER = ['english', 'hindi', 'hinglish'] + +def _sample_clip_choices() -> list[tuple[str, str]]: + manifest = _load_sample_manifest() + by_bucket: dict[str, list[dict]] = {b: [] for b in BUCKET_ORDER} + for entry in manifest: + by_bucket.setdefault(entry.get('bucket', 'other'), []).append(entry) + choices = [] + for bucket in BUCKET_ORDER: + for i, entry in enumerate(by_bucket.get(bucket, []), start=1): + label = f"{BUCKET_LABEL.get(bucket, bucket.title())} {i} - {entry['duration_seconds']}s" + choices.append((label, entry['filename'])) + for bucket, entries in by_bucket.items(): + if bucket in BUCKET_ORDER: + continue + for i, entry in enumerate(entries, start=1): + choices.append((f"{bucket.title()} {i} - {entry['duration_seconds']}s", entry['filename'])) + return choices + +def _preset_choices() -> list[str]: + return [live.DISPLAY_NAME.get(p['label'], p['label']) for p in live.public_presets()] + +def _default_active() -> list[str]: + choices = _preset_choices() + return [choices[0]] if choices else [] + +def _param_relevance(active_display_names: list[str]) -> tuple[bool, bool]: + display_to_internal = {v: k for k, v in live.DISPLAY_NAME.items()} + labels = [display_to_internal[n] for n in active_display_names or [] if n in display_to_internal] + acoustic_relevant = any((live.uses_acoustic_weight(label) for label in labels)) + semantic_relevant = any((live.uses_semantic_temperature(label) for label in labels)) + return (acoustic_relevant, semantic_relevant) + +def build_app() -> gr.Blocks: + with gr.Blocks(title='Turn Detection - Live Dashboard') as demo: + gr.Markdown('# Turn Detection - Live Dashboard\nSpeak, or replay a clip, and watch how different models judge whether the speaker is **done talking** vs. **still going** - plotted directly against the waveform as audio arrives.') + session_state = gr.State(live.new_session_state) + with gr.Row(): + active_presets = gr.CheckboxGroup(choices=_preset_choices(), value=_default_active(), label='Models to compare') + with gr.Row(): + threshold_slider = gr.Slider(0.0, 1.0, value=live.DEFAULT_DECISION_THRESHOLD, step=0.01, label='Decision threshold - probability above this = "complete"') + cadence_slider = gr.Slider(0, 5000, value=live.DEFAULT_CADENCE_MS, step=100, label='Update cadence (ms) - how often each model re-checks (slower models may still lag behind this)') + smoothing_slider = gr.Slider(0.0, 0.9, value=0.5, step=0.05, label="Smoothing - damps short swings on the chart (raw points still shown faintly; doesn't change what's logged)") + with gr.Row(): + acoustic_weight_slider = gr.Slider(0.0, 1.0, value=0.6, label='Acoustic weight - audio tone vs. sentence grammar (fusion models only)', interactive=False) + temperature_slider = gr.Slider(0.0, 1.0, value=0.2, label='Semantic temperature - how deterministic the language judgment is (LLM-based models only)', interactive=False) + with gr.Row(): + with gr.Column(scale=1): + gr.Markdown('### Audio') + mic = gr.Audio(sources=['microphone'], streaming=True, type='numpy', label='Record') + gr.Markdown('*Recordings are saved privately to improve the model - never played back or shown to others.*') + clear_btn = gr.Button('Clear / reset') + gr.Markdown('**...or replay a clip in real time** (paced to its real duration, not dumped in at once) - 10 real English, 10 Hindi, 10 Hinglish') + replay_clip_dropdown = gr.Dropdown(choices=_sample_clip_choices(), value=None, label='Sample clip') + replay_upload = gr.Audio(sources=['upload'], type='filepath', label='...or upload a recording') + replay_btn = gr.Button('Replay in real time') + with gr.Column(scale=2): + gr.Markdown('### Waveform + live probability, on one timeline') + chart = gr.Plot(value=live.render_chart(live.new_session_state(), [], live.DEFAULT_DECISION_THRESHOLD), label=None) + with gr.Accordion('History (this session)', open=False): + log_table = gr.Dataframe(headers=LOG_HEADERS, value=[], label=None) + active_presets.change(lambda active: (gr.update(interactive=_param_relevance(active)[0]), gr.update(interactive=_param_relevance(active)[1])), inputs=active_presets, outputs=[acoustic_weight_slider, temperature_slider]) + + def replay_clip(clip_filename, uploaded_path, active_display_names, acoustic_weight, temperature, threshold, cadence_ms, smoothing): + path = uploaded_path or (str(config.SAMPLE_CLIPS_DIR / clip_filename) if clip_filename else None) + if not path: + yield (live.new_session_state(), live.render_chart(live.new_session_state(), [], threshold, smoothing), []) + return + audio, sr = sf.read(path, dtype='float32') + if audio.ndim > 1: + audio = audio.mean(axis=1) + state = live.new_session_state() + for chunk in live.chunk_audio(audio, sr, chunk_seconds=1.0): + chunk_duration_s = len(chunk) / sr + state, fig, log_rows = live.process_chunk(state, (sr, chunk), active_display_names, acoustic_weight, temperature, threshold, cadence_ms, smoothing) + yield (state, fig, log_rows) + time.sleep(chunk_duration_s) + replay_btn.click(replay_clip, inputs=[replay_clip_dropdown, replay_upload, active_presets, acoustic_weight_slider, temperature_slider, threshold_slider, cadence_slider, smoothing_slider], outputs=[session_state, chart, log_table]) + mic.stream(fn=live.process_chunk, inputs=[session_state, mic, active_presets, acoustic_weight_slider, temperature_slider, threshold_slider, cadence_slider, smoothing_slider], outputs=[session_state, chart, log_table], stream_every=1.0, time_limit=None) + clear_btn.click(live.clear_session, inputs=session_state, outputs=[session_state, chart, log_table]) + return demo \ No newline at end of file diff --git a/frontend/live.py b/frontend/live.py new file mode 100644 index 0000000000000000000000000000000000000000..c0bf6450850a03420173f9355396c2f935246953 --- /dev/null +++ b/frontend/live.py @@ -0,0 +1,198 @@ +from __future__ import annotations +import json +import time +import uuid +from pathlib import Path +from typing import Any +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import numpy as np +import soundfile as sf +import config +from backend.orchestrator import run_pipeline +from backend.presets import list_presets +from backend.types import StageUnavailableError +MAX_BUFFER_SECONDS = 60 +DEFAULT_DECISION_THRESHOLD = 0.5 +DEFAULT_CADENCE_MS = 1000 +PUBLIC_PRESET_ORDER = ['Smart Turn v3.2 (zero-shot)', 'Whisper-Tiny + Mean-Pool + Linear (trained)', 'Whisper-Tiny + Mean-Pool + MLP (trained)', 'Whisper-Tiny + Cross-Attn + Linear (trained)', 'TEN Turn Detection (prompted)', 'LiveKit End-of-Turn Detector (zero-shot)', 'Acoustic + Semantic Fusion (rule-based)', 'FastTurn Streaming (reimplemented)'] +DISPLAY_NAME = {'Smart Turn v3.2 (zero-shot)': 'Smart Turn v3.2', 'Whisper-Tiny + Mean-Pool + Linear (trained)': 'Whisper + Linear Head', 'Whisper-Tiny + Mean-Pool + MLP (trained)': 'Whisper + MLP Head', 'Whisper-Tiny + Cross-Attn + Linear (trained)': 'Whisper + Attention Head', 'TEN Turn Detection (prompted)': 'Semantic (LLM-based)', 'LiveKit End-of-Turn Detector (zero-shot)': 'LiveKit EOU Detector', 'Acoustic + Semantic Fusion (rule-based)': 'Acoustic + Semantic Fusion', 'FastTurn Streaming (reimplemented)': 'Streaming Semantic'} +CURVE_COLORS = ['#58a6ff', '#3fb950', '#f0883e', '#d2a8ff', '#f778ba', '#79c0ff', '#ffa657'] +LIVE_RUNS_DIR = config.LIVE_RUNS_DIR + +def public_presets() -> list[dict]: + by_label = {p['label']: p for p in list_presets()} + return [by_label[label] for label in PUBLIC_PRESET_ORDER if label in by_label and by_label[label]['available']] + +def uses_acoustic_weight(label: str) -> bool: + by_label = {p['label']: p for p in list_presets()} + preset = by_label.get(label) + return bool(preset and preset['config'].fusion != 'off') + +def uses_semantic_temperature(label: str) -> bool: + by_label = {p['label']: p for p in list_presets()} + preset = by_label.get(label) + return bool(preset and preset['config'].semantic != 'off') + +def new_session_state() -> dict[str, Any]: + session_id = str(uuid.uuid4())[:8] + return {'session_id': session_id, 'buffer': np.zeros(0, dtype=np.float32), 'last_run': {}, 'started_at': time.time(), 'log': [], '_last_result': {}} + +def log_path(session_id: str) -> Path: + LIVE_RUNS_DIR.mkdir(parents=True, exist_ok=True) + return LIVE_RUNS_DIR / f'{session_id}.jsonl' + +def audio_path(session_id: str) -> Path: + LIVE_RUNS_DIR.mkdir(parents=True, exist_ok=True) + return LIVE_RUNS_DIR / f'{session_id}.wav' + +def _save_chunk_to_disk(session_id: str, chunk: np.ndarray) -> None: + if chunk.size == 0: + return + path = audio_path(session_id) + if path.exists(): + with sf.SoundFile(path, mode='r+') as f: + f.seek(0, sf.SEEK_END) + f.write(chunk) + else: + with sf.SoundFile(path, mode='w', samplerate=config.SAMPLE_RATE, channels=1, subtype='PCM_16') as f: + f.write(chunk) + +def _append_chunk(buffer: np.ndarray, chunk_sr: int, chunk: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + chunk = np.asarray(chunk) + if chunk.dtype.kind == 'i': + chunk = chunk.astype(np.float32) / 32768.0 + if chunk.ndim > 1: + chunk = chunk.mean(axis=1) + chunk = chunk.astype(np.float32) + if chunk_sr != config.SAMPLE_RATE: + import librosa + chunk = librosa.resample(chunk, orig_sr=chunk_sr, target_sr=config.SAMPLE_RATE) + buffer = np.concatenate([buffer, chunk]) + max_samples = MAX_BUFFER_SECONDS * config.SAMPLE_RATE + if len(buffer) > max_samples: + buffer = buffer[-max_samples:] + return (buffer, chunk) + +def _downsample_envelope(buffer: np.ndarray, sample_rate: int, target_points: int=1500) -> tuple[np.ndarray, np.ndarray]: + n = len(buffer) + if n == 0: + return (np.array([0.0]), np.array([0.0])) + bucket = max(1, n // target_points) + trimmed = buffer[:n - n % bucket] if bucket > 1 else buffer + if len(trimmed) == 0: + trimmed = buffer + bucket = 1 + reshaped = trimmed.reshape(-1, bucket) if bucket > 1 else trimmed.reshape(-1, 1) + envelope = np.abs(reshaped).max(axis=1) + times = np.arange(len(envelope)) * bucket / sample_rate + return (times, envelope) + +def _ema_smooth(ys: list[float], smoothing: float) -> list[float]: + if not ys or smoothing <= 0: + return list(ys) + smoothed = [ys[0]] + for y in ys[1:]: + smoothed.append(smoothing * smoothed[-1] + (1 - smoothing) * y) + return smoothed + +def render_chart(state: dict[str, Any], active_labels: list[str], threshold: float, smoothing: float=0.0): + buffer = state['buffer'] + buffer_seconds = len(buffer) / config.SAMPLE_RATE + x_max = max(buffer_seconds, 1.0) + fig, (ax_wave, ax_prob) = plt.subplots(2, 1, figsize=(9, 5), sharex=True, height_ratios=[1, 2.5], gridspec_kw={'hspace': 0.08}, constrained_layout=True) + fig.patch.set_facecolor('#0d0d0d') + for ax in (ax_wave, ax_prob): + ax.set_facecolor('#0d0d0d') + for spine in ax.spines.values(): + spine.set_color('#333') + ax.tick_params(colors='#888', labelsize=8) + times, envelope = _downsample_envelope(buffer, config.SAMPLE_RATE) + ax_wave.fill_between(times, -envelope, envelope, color='#58a6ff', alpha=0.6, linewidth=0) + ax_wave.set_yticks([]) + ax_wave.set_ylabel('audio', color='#888', fontsize=9) + ax_wave.set_xlim(0, x_max) + verdict_only_lines = [] + color_i = 0 + for label in active_labels: + display_name = DISPLAY_NAME.get(label, label) + history = [e for e in state['log'] if e['model'] == display_name] + if not history: + continue + has_probability = any((e['probability'] is not None for e in history)) + if not has_probability: + last = history[-1] + verdict_only_lines.append(f"{display_name}: {(last['decision'] or '-').upper()}") + continue + xs = [e['buffer_seconds'] for e in history if e['probability'] is not None] + ys_raw = [e['probability'] for e in history if e['probability'] is not None] + if xs and xs[-1] < buffer_seconds: + xs = xs + [buffer_seconds] + ys_raw = ys_raw + [ys_raw[-1]] + ys = _ema_smooth(ys_raw, smoothing) + color = CURVE_COLORS[color_i % len(CURVE_COLORS)] + color_i += 1 + current = ys[-1] if ys else None + current_decision = 'COMPLETE' if current is not None and current > threshold else 'INCOMPLETE' + ax_prob.step(xs, ys, where='post', color=color, linewidth=2, label=f'{display_name} - {current_decision} ({current * 100:.0f}%)' if current is not None else display_name) + ax_prob.scatter(xs, ys_raw, color=color, s=10, zorder=3, alpha=0.35 if smoothing > 0 else 1.0) + ax_prob.axhline(threshold, color='#fff', linewidth=1, linestyle='--', alpha=0.6) + ax_prob.text(x_max, threshold, f' threshold {threshold:.0%}', color='#ccc', fontsize=8, va='center') + ax_prob.set_ylim(-0.03, 1.03) + ax_prob.set_xlim(0, x_max) + ax_prob.set_ylabel('probability complete', color='#888', fontsize=9) + ax_prob.set_xlabel('seconds into this recording', color='#888', fontsize=9) + if ax_prob.get_legend_handles_labels()[0]: + legend = ax_prob.legend(loc='lower left', fontsize=8, framealpha=0.85, facecolor='#161616', labelcolor='#eee') + legend.get_frame().set_edgecolor('#333') + if verdict_only_lines: + fig.text(0.01, 0.005, ' ยท '.join(verdict_only_lines), color='#aaa', fontsize=8) + plt.close(fig) + return fig + +def process_chunk(state: dict[str, Any], stream_chunk, active_display_names: list[str], acoustic_weight: float, temperature: float, threshold: float=DEFAULT_DECISION_THRESHOLD, cadence_ms: float=DEFAULT_CADENCE_MS, smoothing: float=0.0): + if state is None: + state = new_session_state() + display_to_internal = {v: k for k, v in DISPLAY_NAME.items()} + active_labels = [display_to_internal[n] for n in active_display_names or [] if n in display_to_internal] + cadence_s = max(0.0, cadence_ms) / 1000.0 + if stream_chunk is not None: + chunk_sr, chunk_data = stream_chunk + state['buffer'], resampled_chunk = _append_chunk(state['buffer'], chunk_sr, chunk_data) + _save_chunk_to_disk(state['session_id'], resampled_chunk) + buffer = state['buffer'] + buffer_seconds = len(buffer) / config.SAMPLE_RATE + presets_by_label = {p['label']: p for p in list_presets()} + now = time.time() + if buffer_seconds >= 0.2: + for label in active_labels: + preset = presets_by_label.get(label) + if preset is None or not preset['available']: + continue + last_run = state['last_run'].get(label, 0.0) + if now - last_run < cadence_s: + continue + display_name = DISPLAY_NAME.get(label, label) + cfg = preset['config'] + cfg = type(cfg)(**{**cfg.__dict__, 'acoustic_weight': acoustic_weight, 'semantic_temperature': temperature}) + try: + result = run_pipeline(cfg, buffer, sample_rate=config.SAMPLE_RATE, preset_label=label) + state['last_run'][label] = now + log_entry = {'buffer_seconds': round(buffer_seconds, 2), 'model': display_name, 'decision': result.decision, 'probability': result.probability, 'latency_ms': round(result.total_latency_ms, 1)} + state['log'].append(log_entry) + with open(log_path(state['session_id']), 'a') as f: + f.write(json.dumps(log_entry) + '\n') + except StageUnavailableError: + pass + chart = render_chart(state, active_labels, threshold, smoothing) + log_rows = [[e['buffer_seconds'], e['model'], e['decision'], e['probability'], e['latency_ms']] for e in state['log'][-100:]] + return (state, chart, log_rows) + +def clear_session(_state): + return (new_session_state(), render_chart(new_session_state(), [], DEFAULT_DECISION_THRESHOLD), []) + +def chunk_audio(audio: np.ndarray, sample_rate: int, chunk_seconds: float=1.0): + chunk_samples = max(1, int(chunk_seconds * sample_rate)) + for start in range(0, len(audio), chunk_samples): + yield audio[start:start + chunk_samples] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..88fab4dda24c21e304b5e32556e8f5a4b00109da --- /dev/null +++ b/requirements.txt @@ -0,0 +1,123 @@ +aiohappyeyeballs==2.7.1 +aiohttp==3.14.3 +aiosignal==1.4.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.14.2 +attrs==26.1.0 +audioop-lts==0.2.2 ; python_full_version >= '3.13' +brotli==1.2.0 +certifi==2026.7.22 +cffi==2.1.1 +charset-normalizer==3.5.1 +click==8.4.2 +colorama==0.4.6 ; sys_platform == 'win32' +contourpy==1.3.3 +cuda-bindings==13.3.1 ; python_full_version < '3.15' and sys_platform == 'linux' +cuda-pathfinder==1.6.1 ; python_full_version < '3.15' and sys_platform == 'linux' +cuda-toolkit==13.0.3.0 ; sys_platform == 'linux' +cycler==0.12.1 +datasets==5.0.1 +decorator==5.3.1 +dill==0.4.1 +fastapi==0.141.1 +filelock==3.32.4 +flatbuffers==25.12.19 +fonttools==4.63.0 +frozenlist==1.8.0 +fsspec==2026.6.0 +gradio==6.25.0 +gradio-client==2.6.0 +groovy==0.1.2 +h11==0.16.0 +hf-gradio==0.4.1 +hf-xet==1.6.0 ; platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==1.28.0 +idna==3.19 +jinja2==3.1.6 +joblib==1.5.3 +kiwisolver==1.5.0 +lazy-loader==0.5 +librosa==1.0.0 +llvmlite==0.49.0 +markdown-it-py==4.2.0 +markupsafe==3.0.3 +matplotlib==3.11.1 +mdurl==0.1.2 +mpmath==1.3.0 +msgpack==1.2.1 +multidict==6.7.1 +multiprocess==0.70.19 +narwhals==2.25.0 +networkx==3.6.1 +numba==0.67.0 +numpy==2.5.2 +nvidia-cublas==13.1.1.3 ; sys_platform == 'linux' +nvidia-cuda-cupti==13.0.85 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cuda-nvrtc==13.0.88 ; sys_platform == 'linux' +nvidia-cuda-runtime==13.0.96 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cudnn-cu13==9.20.0.48 ; sys_platform == 'linux' +nvidia-cufft==12.0.0.61 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cufile==1.15.1.6 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-curand==10.4.0.35 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cusolver==12.0.4.66 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cusparse==12.6.3.3 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-cusparselt-cu13==0.8.1 ; sys_platform == 'linux' +nvidia-nccl-cu13==2.29.7 ; sys_platform == 'linux' +nvidia-nvjitlink==13.3.33 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +nvidia-nvshmem-cu13==3.4.5 ; sys_platform == 'linux' +nvidia-nvtx==13.0.85 ; (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +onnxruntime==1.29.0 +orjson==3.12.0 +packaging==26.3 +pandas==3.0.5 +pillow==12.3.0 +platformdirs==4.11.3 +pooch==1.9.0 +propcache==0.5.2 +protobuf==7.36.0 +pyarrow==25.0.1 +pycparser==3.0 ; implementation_name != 'PyPy' +pydantic==2.13.4 +pydantic-core==2.46.4 +pydub==0.25.1 +pygments==2.21.0 +pyparsing==3.3.2 +python-dateutil==2.9.0.post0 +python-multipart==0.0.32 +pytz==2026.3.post1 +pyyaml==6.0.3 +regex==2026.7.19 +requests==2.34.2 +rich==15.0.0 +safehttpx==0.1.7 +safetensors==0.8.0 +scikit-learn==1.9.0 +scipy==1.18.1 +semantic-version==2.10.0 +setuptools==84.0.0 +shellingham==1.5.4 +six==1.17.0 +soundfile==0.14.0 +soxr==1.1.0 +starlette==1.6.0 +sympy==1.14.0 +threadpoolctl==3.6.0 +tokenizers==0.22.2 +tomlkit==0.14.0 +torch==2.13.0 +torchaudio==2.11.0 +tqdm==4.70.0 +transformers==5.15.1 +triton==3.7.1 ; python_full_version < '3.15' and sys_platform == 'linux' +typer==0.27.1 +typing-extensions==4.16.0 +typing-inspection==0.4.4 +tzdata==2026.3 ; sys_platform == 'emscripten' or sys_platform == 'win32' +urllib3==2.7.0 +uvicorn==0.52.4 +webrtcvad-wheels==2.0.14 +xxhash==4.0.1 +yarl==1.24.5