half-duplex / app.py
Nampfiev1995's picture
Update SmartTurn model tabs
33b3304 verified
Raw
History Blame Contribute Delete
49.6 kB
#!/usr/bin/env python3
from __future__ import annotations
import json
import os
from dataclasses import dataclass
from pathlib import Path
from typing import Any
os.environ.setdefault("MPLCONFIGDIR", "/tmp/matplotlib")
import gradio as gr
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import onnxruntime as ort
import soundfile as sf
import torch
from scipy.signal import resample_poly
ROOT = Path(__file__).resolve().parent
MODELS = ROOT / "models"
IS_HF_SPACE = bool(os.environ.get("SPACE_ID"))
os.environ.setdefault("HF_HOME", "/tmp/huggingface")
os.environ.setdefault("TRANSFORMERS_CACHE", str(Path(os.environ["HF_HOME"]) / "transformers"))
RUNS = Path(os.environ.get("HALF_DUPLEX_RUN_DIR", "/tmp/half_duplex_runs" if IS_HF_SPACE else str(ROOT / "runs")))
DEFAULT_REPO = Path("/Utilisateurs/tnguye28/vad-lstm")
DEFAULT_ENROLL = DEFAULT_REPO / "debug" / "audio (6).wav"
DEFAULT_MIC = DEFAULT_REPO / "debug" / "audio (7).wav"
DEFAULT_ASSISTANT = DEFAULT_REPO / "debug" / "test-target-spk4.wav"
DEFAULT_PVAD_ONNX = MODELS / "pvad_core.onnx"
DEFAULT_PVAD_H256_ONNX = MODELS / "pvad_core_h256.onnx"
DEFAULT_SILERO_JIT = MODELS / "silero_vad.jit"
DEFAULT_SMARTTURN_ONNX = MODELS / "smartturn-v3.1.onnx"
LOCAL_SOTA_PREVBEST_CK50_INT8 = MODELS / "sota_prevbest_incw110_ck50_staticcalib8.onnx"
LOCAL_SOTA_PREVBEST_CK100_INT8 = MODELS / "sota_prevbest_incw110_ck100_staticcalib8.onnx"
LOCAL_SOTA_PREVBEST_CK150_INT8 = MODELS / "sota_prevbest_incw110_ck150_staticcalib8.onnx"
LOCAL_SOTA_HARDNEG4K_CK50_INT8 = MODELS / "sota_hardneg4k_v2_incw110_ck50_staticcalib8.onnx"
SOTA_ONNX_ROOT = Path("/Utilisateurs/tnguye28/smartturn-vn/outputs/lumi_turn/onnx_exports")
REMOTE_SOTA_PREVBEST_CK50_INT8 = SOTA_ONNX_ROOT / "sota_prevbest_incw110_ck50_staticcalib8" / "model_int8_static_calib8.onnx"
REMOTE_SOTA_PREVBEST_CK100_INT8 = SOTA_ONNX_ROOT / "sota_prevbest_incw110_ck100_staticcalib8" / "model_int8_static_calib8.onnx"
REMOTE_SOTA_PREVBEST_CK150_INT8 = SOTA_ONNX_ROOT / "sota_prevbest_incw110_ck150_staticcalib8" / "model_int8_static_calib8.onnx"
REMOTE_SOTA_HARDNEG4K_CK50_INT8 = SOTA_ONNX_ROOT / "sota_hardneg4k_v2_incw110_ck50_staticcalib8" / "model_int8_static_calib8.onnx"
DUALTURN_MODEL_ID = "anyreach-ai/dualturn-qwen2.5-mimi-0.5B"
DEFAULT_DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
STATE_COLORS = {
"ACTIVE": "#d62828",
"HOLD": "#f4a261",
"SOFT_END": "#2563eb",
"END": "#2a9d8f",
"INTERRUPT": "#7b2cbf",
"UNKNOWN": "#8d99ae",
}
def load_wav_16k(path: str | Path | None) -> np.ndarray:
if path is None:
return np.zeros(0, dtype=np.float32)
audio, sr = sf.read(str(path), dtype="float32", always_2d=False)
if audio.ndim > 1:
audio = np.mean(audio, axis=1)
audio = np.asarray(audio, dtype=np.float32).reshape(-1)
if sr != 16000:
gcd = np.gcd(sr, 16000)
audio = resample_poly(audio, 16000 // gcd, sr // gcd).astype(np.float32)
peak = float(np.max(np.abs(audio))) if len(audio) else 0.0
if peak > 1.0:
audio = audio / peak
return audio
def default_audio_value(path: Path) -> str | None:
return str(path) if path.exists() else None
def prefer_existing(local_path: Path, remote_path: Path) -> Path:
return local_path if local_path.exists() else remote_path
def write_wav(path: Path, audio: np.ndarray, sample_rate: int = 16000) -> str:
path.parent.mkdir(parents=True, exist_ok=True)
sf.write(str(path), np.asarray(audio, dtype=np.float32), sample_rate)
return str(path)
def make_ort_session(path: str | Path) -> ort.InferenceSession:
path = str(path).strip()
opts = ort.SessionOptions()
opts.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
opts.intra_op_num_threads = 1
opts.inter_op_num_threads = 1
opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
return ort.InferenceSession(path, sess_options=opts, providers=["CPUExecutionProvider"])
def frame_audio(audio: np.ndarray, frame_size: int = 512) -> np.ndarray:
if len(audio) == 0:
return np.zeros((1, frame_size), dtype=np.float32)
n = int(np.ceil(len(audio) / frame_size))
padded = np.zeros(n * frame_size, dtype=np.float32)
padded[: len(audio)] = audio
return padded.reshape(n, frame_size)
def resample_audio(audio: np.ndarray, src_sr: int, dst_sr: int) -> np.ndarray:
audio = np.asarray(audio, dtype=np.float32).reshape(-1)
if src_sr == dst_sr:
return audio
gcd = np.gcd(src_sr, dst_sr)
return resample_poly(audio, dst_sr // gcd, src_sr // gcd).astype(np.float32)
class PvadOnnx:
def __init__(self, pvad_path: str | Path, silero_path: str | Path):
self.session = make_ort_session(pvad_path)
self.num_layers, self.hidden_dim = self._infer_recurrent_shape()
jit = torch.jit.load(str(silero_path), map_location="cpu")
self.silero = jit._model if hasattr(jit, "_model") else jit
self.silero.eval()
def _infer_recurrent_shape(self) -> tuple[int, int]:
inputs = {inp.name: inp for inp in self.session.get_inputs()}
h0 = inputs.get("h0")
if h0 is None or len(h0.shape) != 3:
raise ValueError("PVAD ONNX must expose h0 input with shape [layers, batch, hidden]")
layers, _batch, hidden = h0.shape
if not isinstance(layers, int) or not isinstance(hidden, int):
raise ValueError(f"PVAD ONNX h0 shape must have static layers/hidden dims, got {h0.shape}")
return int(layers), int(hidden)
def silero_scores(self, frames: np.ndarray) -> np.ndarray:
context = torch.zeros(1, 64, dtype=torch.float32)
state = torch.zeros(2, 1, 128, dtype=torch.float32)
scores = []
with torch.no_grad():
for frame in frames:
frame_t = torch.from_numpy(frame.reshape(1, 512).astype(np.float32))
score, state = self.silero(torch.cat([context, frame_t], dim=1), state)
context = frame_t[:, -64:]
scores.append(float(score.reshape(-1)[0].item()))
return np.asarray(scores, dtype=np.float32).reshape(-1, 1)
def run_core(
self,
frames: np.ndarray,
target_vector: np.ndarray,
vad_scores: np.ndarray,
) -> tuple[np.ndarray, np.ndarray]:
h = np.zeros((self.num_layers, 1, self.hidden_dim), dtype=np.float32)
c = np.zeros((self.num_layers, 1, self.hidden_dim), dtype=np.float32)
target = target_vector.reshape(1, 16).astype(np.float32)
probs = []
embeds = []
for frame, vad_score in zip(frames, vad_scores):
final_prob, _raw_prob, embed, h, c = self.session.run(
None,
{
"frame_pcm": frame.reshape(1, 512).astype(np.float32),
"target_vector": target,
"vad_score": vad_score.reshape(1, 1).astype(np.float32),
"h0": h,
"c0": c,
},
)
probs.append(final_prob[0])
embeds.append(embed[0])
return np.asarray(probs, dtype=np.float32), np.asarray(embeds, dtype=np.float32)
def target_vector(self, enroll_audio: np.ndarray) -> np.ndarray:
frames = frame_audio(enroll_audio)
scores = self.silero_scores(frames)
_probs, embeds = self.run_core(frames, np.zeros(16, dtype=np.float32), scores)
weights = scores.reshape(-1, 1)
pooled = np.sum(embeds * weights, axis=0) / (float(np.sum(weights)) + 1e-8)
norm = float(np.linalg.norm(pooled))
return (pooled / max(norm, 1e-8)).astype(np.float32)
def predict_target_probs(self, mic_audio: np.ndarray, enroll_audio: np.ndarray) -> np.ndarray:
max_val = float(np.max(np.abs(mic_audio)) + 1e-8) if len(mic_audio) else 1.0
frames = frame_audio(mic_audio / max_val)
target = self.target_vector(enroll_audio)
scores = self.silero_scores(frames)
probs, _embeds = self.run_core(frames, target, scores)
return probs[:, 0].astype(np.float32)
class SmartTurnOnnx:
def __init__(self, model_path: str | Path):
from transformers import WhisperFeatureExtractor
self.session = make_ort_session(model_path)
self.feature_extractor = WhisperFeatureExtractor(chunk_length=8)
def predict_prob(self, audio_16k: np.ndarray) -> float:
samples = np.asarray(audio_16k, dtype=np.float32).reshape(-1)
max_samples = 8 * 16000
if len(samples) > max_samples:
samples = samples[-max_samples:]
elif len(samples) < max_samples:
samples = np.pad(samples, (max_samples - len(samples), 0), mode="constant")
inputs = self.feature_extractor(
samples,
sampling_rate=16000,
return_tensors="np",
padding="max_length",
max_length=max_samples,
truncation=True,
do_normalize=True,
)
features = inputs.input_features.squeeze(0).astype(np.float32)[None, ...]
outputs = self.session.run(None, {"input_features": features})
return float(outputs[0][0].item())
class DualTurnHF:
def __init__(self, model_id: str, device: str):
from transformers import AutoModel
use_device = "cuda" if device == "cuda" and torch.cuda.is_available() else "cpu"
self.device = torch.device(use_device)
self.model = AutoModel.from_pretrained(model_id, trust_remote_code=True)
self.model.to(self.device)
self.model.eval()
@torch.no_grad()
def predict_channels(self, ch0_16k: np.ndarray, ch1_16k: np.ndarray) -> dict[str, np.ndarray]:
ch0 = resample_audio(ch0_16k, 16000, 24000)
ch1 = resample_audio(ch1_16k, 16000, 24000)
n = min(len(ch0), len(ch1))
if n <= 0:
ch0 = np.zeros(1, dtype=np.float32)
ch1 = np.zeros(1, dtype=np.float32)
else:
ch0 = ch0[:n]
ch1 = ch1[:n]
stereo = torch.from_numpy(np.stack([ch0, ch1], axis=0)).to(self.device)
out = self.model(stereo, sr=24000)
fvad = out.fvad_probs.detach().float().cpu().numpy()
if fvad.ndim == 3:
fvad = fvad[0]
return {
"vad": self._user_np(out.vad_probs),
"hold": self._user_np(out.hold_probs),
"eot": self._user_np(out.eot_probs),
"fvad_240": np.asarray(fvad[:, 0], dtype=np.float32).reshape(-1),
"fvad_480": np.asarray(fvad[:, 1], dtype=np.float32).reshape(-1),
"fvad_960": np.asarray(fvad[:, 2], dtype=np.float32).reshape(-1),
"fvad_2000": np.asarray(fvad[:, 3], dtype=np.float32).reshape(-1),
}
@staticmethod
def _user_np(tensor: torch.Tensor) -> np.ndarray:
arr = tensor.detach().float().cpu().numpy()
if arr.ndim == 3:
arr = arr[0]
if arr.ndim == 2:
arr = arr[:, 0]
return np.asarray(arr, dtype=np.float32).reshape(-1)
def dualturn_state(row: dict[str, float]) -> str:
if row["vad"] >= 0.5:
return "ACTIVE"
if row["hold"] >= 0.5:
return "HOLD"
if max(row["fvad_240"], row["fvad_480"], row["fvad_960"], row["fvad_2000"]) >= 0.5:
return "CONTINUE"
if row["eot"] >= 0.6 and row["hold"] < 0.4 and row["fvad_480"] < 0.35 and row["fvad_960"] < 0.35:
return "END"
return "UNKNOWN"
def latest_dualturn_row(outputs: dict[str, np.ndarray], time_ms: int) -> dict[str, Any]:
n = min(len(v) for v in outputs.values())
idx = max(0, n - 1)
row = {k: float(v[idx]) if len(v) else 0.0 for k, v in outputs.items()}
row["time_ms"] = int(time_ms)
row["dualturn_state"] = dualturn_state(row)
return row
def make_pvad_target_audio(audio: np.ndarray, probs_32ms: np.ndarray, threshold: float) -> np.ndarray:
out = np.zeros_like(audio, dtype=np.float32)
frame = 512
for idx, prob in enumerate(probs_32ms):
start = idx * frame
end = min(len(audio), start + frame)
if end <= start:
break
if prob >= threshold:
out[start:end] = audio[start:end]
return out
def pvad_active_between(probs_32ms: np.ndarray, start_sample: int, end_sample: int, threshold: float) -> bool:
start_idx = max(0, start_sample // 512)
end_idx = min(len(probs_32ms), int(np.ceil(end_sample / 512)))
if end_idx <= start_idx:
return False
return bool(np.max(probs_32ms[start_idx:end_idx]) >= threshold)
def frame_is_active(audio: np.ndarray, threshold: float = 0.01) -> bool:
if len(audio) == 0:
return False
return float(np.sqrt(np.mean(np.asarray(audio, dtype=np.float32) ** 2))) >= threshold
def active_audio_sec(target_audio: np.ndarray, start: int, end: int) -> float:
part = target_audio[start:end]
return float(np.count_nonzero(np.abs(part) > 1e-8) / 16000.0)
def append_row(rows: list[dict[str, Any]], row: dict[str, Any]) -> None:
if rows and rows[-1]["state"] == row["state"] and rows[-1]["source"] == row["source"]:
return
rows.append(row)
@dataclass
class RunResult:
rows: list[dict[str, Any]]
pvad_probs: np.ndarray
pvad_target_audio: np.ndarray
assistant_track: np.ndarray
smart_probs: list[tuple[float, float]]
cuts: list[dict[str, Any]]
def run_pipeline(
enroll_path: str,
mic_path: str,
assistant_path: str | None,
*,
mode: str,
smartturn_threshold: float,
pvad_threshold: float,
pvad_model_path: str,
smartturn_model_path: str,
min_active_target_ms: float,
silence_fallback_ms: float,
asr_cut_silence_ms: float,
model_check_interval_ms: float,
append_assistant_after_end: bool,
assistant_max_playback_sec: float,
device: str,
) -> RunResult:
enroll = load_wav_16k(enroll_path)
mic = load_wav_16k(mic_path)
assistant = load_wav_16k(assistant_path) if assistant_path else np.zeros(0, dtype=np.float32)
if assistant_max_playback_sec > 0:
assistant = assistant[: int(round(assistant_max_playback_sec * 16000))]
pvad = PvadOnnx(pvad_model_path, DEFAULT_SILERO_JIT)
smartturn = SmartTurnOnnx(smartturn_model_path)
dualturn = DualTurnHF(DUALTURN_MODEL_ID, device)
pvad_probs = pvad.predict_target_probs(mic, enroll)
target_audio = mic if mode == "raw" else make_pvad_target_audio(mic, pvad_probs, pvad_threshold)
assistant_track = np.zeros_like(mic, dtype=np.float32)
activity_mask = np.zeros_like(mic, dtype=np.float32)
frame_samples = int(round(0.080 * 16000))
check_frames = max(1, int(round(model_check_interval_ms / 80.0)))
total_frames = max(1, int(np.ceil(len(mic) / frame_samples)))
min_active_sec = min_active_target_ms / 1000.0
silence_fallback_frames = 0 if silence_fallback_ms <= 0 else max(1, int(round(silence_fallback_ms / 80.0)))
asr_cut_silence_frames = 0 if asr_cut_silence_ms <= 0 else max(1, int(round(asr_cut_silence_ms / 80.0)))
rows: list[dict[str, Any]] = []
smart_probs: list[tuple[float, float]] = []
cuts: list[dict[str, Any]] = []
turn_start: int | None = None
asr_cut_start: int | None = None
last_soft_cut_end: int | None = None
last_check_idx: int | None = None
silence_frames = 0
assistant_playing = False
assistant_pos = 0
append_row(rows, {"time_sec": 0.0, "state": "UNKNOWN", "source": "idle", "smartturn": 0.0, "dualturn": "UNKNOWN", "assistant": False})
for idx in range(total_frames):
start = idx * frame_samples
end = min(len(mic), start + frame_samples)
if end <= start:
break
time_sec = start / 16000.0
assistant_active = assistant_playing and assistant_pos < len(assistant)
if assistant_active:
take = min(end - start, len(assistant) - assistant_pos)
assistant_track[start : start + take] += assistant[assistant_pos : assistant_pos + take]
assistant_pos += take
if assistant_pos >= len(assistant):
assistant_playing = False
ch0_frame = target_audio[start:end]
active = frame_is_active(ch0_frame, threshold=0.01)
if active:
activity_mask[start:end] = 1.0
if active and turn_start is None:
turn_start = start
asr_cut_start = start
last_soft_cut_end = None
last_check_idx = None
silence_frames = 0
elif active:
if asr_cut_start is None:
asr_cut_start = start
last_soft_cut_end = None
silence_frames = 0
elif turn_start is not None:
silence_frames += 1
if turn_start is None:
continue
periodic_due = last_check_idx is None or (idx - last_check_idx) >= check_frames
silence_due = silence_fallback_frames > 0 and silence_frames >= silence_fallback_frames
asr_flush_due = asr_cut_silence_frames > 0 and silence_frames >= asr_cut_silence_frames
if not periodic_due and not silence_due and not asr_flush_due:
continue
buffer_ch0 = target_audio[turn_start:end].copy()
buffer_ch0 *= activity_mask[turn_start:end]
buffer_ch1 = assistant_track[turn_start:end]
dual_outputs = dualturn.predict_channels(buffer_ch0, buffer_ch1)
dual = latest_dualturn_row(dual_outputs, int(round(time_sec * 1000)))
smartturn_due = dual["dualturn_state"] in {"HOLD", "END", "UNKNOWN"} or (
(silence_due or asr_flush_due) and dual["dualturn_state"] != "ACTIVE"
)
smart_prob = smartturn.predict_prob(buffer_ch0) if smartturn_due else 0.0
smart_probs.append((time_sec, smart_prob))
last_check_idx = idx
state = "HOLD" if dual["dualturn_state"] in {"CONTINUE", "END"} else dual["dualturn_state"]
source = "dualturn"
active_sec = float(np.sum(activity_mask[turn_start:end] > 0.0) / 16000.0)
future_voice_count = int(float(dual.get("fvad_480", 0.0)) >= 0.5) + int(float(dual.get("fvad_960", 0.0)) >= 0.5)
smart_end = smartturn_due and smart_prob >= smartturn_threshold
if assistant_active and state == "ACTIVE":
state = "INTERRUPT"
source = "assistant_overlap"
assistant_playing = False
assistant_pos = 0
elif smart_end:
if future_voice_count >= 2:
state = "HOLD"
source = "future_voice_guard"
elif future_voice_count == 1:
state = "SOFT_END"
source = "smartturn_soft_partial_future_voice"
else:
state = "END"
source = "smartturn"
elif silence_due and active_sec >= min_active_sec and state != "ACTIVE":
if future_voice_count >= 2:
state = "HOLD"
source = "future_voice_guard"
elif future_voice_count == 1:
state = "SOFT_END"
source = "silence_soft_partial_future_voice"
else:
state = "END"
source = "silence_fallback"
if state == "END" and asr_cut_start is not None and end - asr_cut_start < int(round(min_active_sec * 16000.0)):
state = "HOLD"
source = "short_asr_cut_guard"
append_row(
rows,
{
"time_sec": time_sec,
"state": state,
"source": source,
"smartturn": smart_prob,
"dualturn": dual["dualturn_state"],
"assistant": bool(assistant_active),
"vad": dual["vad"],
"hold": dual["hold"],
"eot": dual["eot"],
"fvad_480": dual["fvad_480"],
"fvad_960": dual["fvad_960"],
},
)
if state == "SOFT_END" and asr_cut_start is not None:
can_emit_soft_cut = (
end - asr_cut_start >= int(round(min_active_sec * 16000.0))
and last_soft_cut_end is None
)
if can_emit_soft_cut:
cut_audio = target_audio[asr_cut_start:end].copy() * activity_mask[asr_cut_start:end]
cuts.append(
{
"start": asr_cut_start / 16000.0,
"end": end / 16000.0,
"duration": (end - asr_cut_start) / 16000.0,
"smartturn": smart_prob,
"dualturn": dual["dualturn_state"],
"vad": dual["vad"],
"hold": dual["hold"],
"eot": dual["eot"],
"fvad_480": dual["fvad_480"],
"fvad_960": dual["fvad_960"],
"note": "soft_end_future_voice",
"audio": cut_audio,
}
)
last_soft_cut_end = end
if state == "END":
if asr_cut_start is not None:
cut_start = asr_cut_start
cut_audio = target_audio[cut_start:end].copy() * activity_mask[cut_start:end]
cuts.append(
{
"start": cut_start / 16000.0,
"end": end / 16000.0,
"duration": (end - cut_start) / 16000.0,
"smartturn": smart_prob,
"dualturn": dual["dualturn_state"],
"vad": dual["vad"],
"hold": dual["hold"],
"eot": dual["eot"],
"fvad_480": dual["fvad_480"],
"fvad_960": dual["fvad_960"],
"note": "hard_end",
"audio": cut_audio,
}
)
if append_assistant_after_end and len(assistant):
assistant_playing = True
assistant_pos = 0
turn_start = None
asr_cut_start = None
last_soft_cut_end = None
last_check_idx = None
silence_frames = 0
elif asr_flush_due and asr_cut_start is not None:
cut_audio = target_audio[asr_cut_start:end].copy() * activity_mask[asr_cut_start:end]
cuts.append(
{
"start": asr_cut_start / 16000.0,
"end": end / 16000.0,
"duration": (end - asr_cut_start) / 16000.0,
"smartturn": smart_prob,
"dualturn": dual["dualturn_state"],
"vad": dual["vad"],
"hold": dual["hold"],
"eot": dual["eot"],
"fvad_480": dual["fvad_480"],
"fvad_960": dual["fvad_960"],
"note": "silence_asr_flush",
"audio": cut_audio,
}
)
asr_cut_start = None
turn_start = None
last_soft_cut_end = None
last_check_idx = None
silence_frames = 0
if asr_cut_start is not None and asr_cut_start < len(mic):
cut_audio = target_audio[asr_cut_start:].copy() * activity_mask[asr_cut_start:]
if turn_start is not None and np.count_nonzero(np.abs(cut_audio) > 1e-8) > 0:
buffer_ch0 = target_audio[turn_start:].copy() * activity_mask[turn_start:]
buffer_ch1 = assistant_track[turn_start:]
dual_outputs = dualturn.predict_channels(buffer_ch0, buffer_ch1)
dual = latest_dualturn_row(dual_outputs, int(round(len(mic) / 16000.0 * 1000)))
smart_prob = smartturn.predict_prob(buffer_ch0)
smart_probs.append((len(mic) / 16000.0, smart_prob))
else:
dual = {"dualturn_state": "FINAL", "vad": 0.0, "hold": 0.0, "eot": 0.0, "fvad_480": 0.0, "fvad_960": 0.0}
smart_prob = 0.0
append_row(
rows,
{
"time_sec": len(mic) / 16000.0,
"state": "FINAL",
"source": "final_flush",
"smartturn": smart_prob,
"dualturn": dual["dualturn_state"],
"assistant": False,
"vad": dual["vad"],
"hold": dual["hold"],
"eot": dual["eot"],
"fvad_480": dual["fvad_480"],
"fvad_960": dual["fvad_960"],
},
)
cuts.append(
{
"start": asr_cut_start / 16000.0,
"end": len(mic) / 16000.0,
"duration": (len(mic) - asr_cut_start) / 16000.0,
"smartturn": smart_prob,
"dualturn": dual["dualturn_state"],
"vad": dual["vad"],
"hold": dual["hold"],
"eot": dual["eot"],
"fvad_480": dual["fvad_480"],
"fvad_960": dual["fvad_960"],
"note": "final_flush",
"audio": cut_audio,
}
)
return RunResult(rows, pvad_probs, target_audio, assistant_track, smart_probs, cuts)
def plot_result(result: RunResult, duration_sec: float, out_path: Path) -> str:
out_path.parent.mkdir(parents=True, exist_ok=True)
times_pvad = np.arange(len(result.pvad_probs), dtype=np.float32) * 0.032
smart_t = [x[0] for x in result.smart_probs]
smart_y = [x[1] for x in result.smart_probs]
fig, axes = plt.subplots(3, 1, figsize=(13, 7.2), sharex=True, gridspec_kw={"height_ratios": [1.3, 1.6, 0.65]})
axes[0].plot(times_pvad, result.pvad_probs, color="#1d4ed8", linewidth=1.2, label="PVAD target")
axes[0].axhline(0.5, color="#64748b", linestyle="--", linewidth=0.8)
axes[0].set_ylim(-0.02, 1.02)
axes[0].legend(loc="upper right")
axes[0].grid(True, alpha=0.25)
dual_times = [r["time_sec"] for r in result.rows if "vad" in r]
for key, color, style in [
("vad", "#d62828", "-"),
("hold", "#f4a261", "-"),
("eot", "#2a9d8f", "-"),
("fvad_480", "#7c3aed", "--"),
("fvad_960", "#0891b2", "--"),
]:
axes[1].plot(
dual_times,
[r.get(key, np.nan) for r in result.rows if "vad" in r],
label=f"DualTurn {key}",
color=color,
linestyle=style,
marker="o",
markersize=2,
linewidth=1.1,
)
axes[1].plot(smart_t, smart_y, color="#2d6a4f", marker="s", markersize=2.5, linewidth=1.2, label="SmartTurn END")
axes[1].axhline(0.9, color="#2d6a4f", linestyle="--", linewidth=0.8, alpha=0.55)
axes[1].set_ylim(-0.02, 1.02)
axes[1].legend(loc="upper right", ncol=2)
axes[1].grid(True, alpha=0.25)
ax = axes[2]
ax.set_ylim(0, 1)
ax.set_yticks([])
ax.set_xlim(0, max(duration_sec, 0.1))
for i, row in enumerate(result.rows):
start = float(row["time_sec"])
end = float(result.rows[i + 1]["time_sec"]) if i + 1 < len(result.rows) else duration_sec
if end <= start:
end = start + 0.08
state = row["state"]
ax.axvspan(start, end, color=STATE_COLORS.get(state, "#8d99ae"), alpha=0.85)
if end - start >= 0.35:
ax.text((start + end) / 2, 0.5, state, ha="center", va="center", fontsize=8, color="white")
ax.axvline(start, color="#111827", linewidth=0.6, alpha=0.35)
ax.set_xlabel("Time (sec)")
fig.tight_layout()
fig.savefig(out_path, dpi=140)
plt.close(fig)
return str(out_path)
def run_gradio(
enroll_audio: str,
mic_audio: str,
assistant_audio: str | None,
mode: str,
smartturn_threshold: float,
pvad_threshold: float,
pvad_model_path: str,
smartturn_model_selection: str,
smartturn_custom_model_path: str,
min_active_target_ms: float,
silence_fallback_ms: float,
asr_cut_silence_ms: float,
model_check_interval_ms: float,
append_assistant_after_end: bool,
assistant_max_playback_sec: float,
device: str,
) -> tuple[str, str, str, str, str, Any, str | None]:
if not enroll_audio:
raise gr.Error("Upload an enrollment audio file.")
if not mic_audio:
raise gr.Error("Upload a mic audio file.")
smartturn_model_path = resolve_smartturn_model_path(smartturn_model_selection, smartturn_custom_model_path)
result = run_pipeline(
enroll_audio,
mic_audio,
assistant_audio,
mode=mode,
smartturn_threshold=float(smartturn_threshold),
pvad_threshold=float(pvad_threshold),
pvad_model_path=pvad_model_path,
smartturn_model_path=smartturn_model_path,
min_active_target_ms=float(min_active_target_ms),
silence_fallback_ms=float(silence_fallback_ms),
asr_cut_silence_ms=float(asr_cut_silence_ms),
model_check_interval_ms=float(model_check_interval_ms),
append_assistant_after_end=bool(append_assistant_after_end),
assistant_max_playback_sec=float(assistant_max_playback_sec),
device=device,
)
RUNS.mkdir(parents=True, exist_ok=True)
mic = load_wav_16k(mic_audio)
timeline = plot_result(result, len(mic) / 16000.0, RUNS / "timeline.png")
target_wav = write_wav(RUNS / "pvad_target_timeline.wav", result.pvad_target_audio)
assistant_wav = write_wav(RUNS / "assistant_channel.wav", result.assistant_track)
mic_with_assistant_wav = write_wav(RUNS / "mic_with_assistant_echo.wav", mic + result.assistant_track)
cut_paths = []
cut_choices = []
cut_dir = RUNS / "turn_cuts"
for idx, cut in enumerate(result.cuts, start=1):
path = cut_dir / f"{idx:03d}_{cut['start']:.2f}_{cut['end']:.2f}.wav"
write_wav(path, cut["audio"])
path_str = str(path)
cut_paths.append(path_str)
label = (
f"{idx:03d} | {cut['start']:.2f}s-{cut['end']:.2f}s | "
f"{cut['duration']:.2f}s | smart={cut['smartturn']:.3f} | "
f"dual={cut.get('dualturn', '')} "
f"vad={cut.get('vad', 0.0):.2f} hold={cut.get('hold', 0.0):.2f} "
f"eot={cut.get('eot', 0.0):.2f} | {cut.get('note', 'cut')}"
)
cut_choices.append((label, path_str))
rows = [
{
"time_sec": round(r["time_sec"], 3),
"state": r["state"],
"source": r["source"],
"smartturn": round(float(r["smartturn"]), 3),
"dualturn": r["dualturn"],
"dual_vad": round(float(r.get("vad", 0.0)), 3),
"dual_hold": round(float(r.get("hold", 0.0)), 3),
"dual_eot": round(float(r.get("eot", 0.0)), 3),
"dual_fvad_480": round(float(r.get("fvad_480", 0.0)), 3),
"dual_fvad_960": round(float(r.get("fvad_960", 0.0)), 3),
"assistant": r["assistant"],
}
for r in result.rows
]
summary = {
"timeline": timeline,
"target_audio": target_wav,
"assistant_channel": assistant_wav,
"mic_with_assistant": mic_with_assistant_wav,
"turn_cuts": cut_paths,
"models": {
"pvad": str(pvad_model_path),
"silero": str(DEFAULT_SILERO_JIT),
"smartturn": str(smartturn_model_path),
"dualturn": DUALTURN_MODEL_ID,
},
"state_counts": {state: sum(1 for r in result.rows if r["state"] == state) for state in STATE_COLORS},
}
first_cut = cut_paths[0] if cut_paths else None
return (
timeline,
json.dumps(summary, indent=2),
json.dumps(rows, indent=2),
target_wav,
mic_with_assistant_wav,
gr.update(choices=cut_choices, value=first_cut),
first_cut,
)
def select_cut_audio(cut_path: str | None) -> str | None:
return cut_path or None
def resolve_smartturn_model_path(selection: str, custom_path: str | None) -> str:
selection = str(selection).strip()
if selection == "custom":
path = str(custom_path or "").strip()
if not path:
raise gr.Error("Paste a SmartTurn ONNX model path or choose one from the dropdown.")
return path
return selection
def plot_smartturn_only(rows: list[dict[str, Any]], duration_sec: float, out_path: Path) -> str:
out_path.parent.mkdir(parents=True, exist_ok=True)
times = [float(r["time_sec"]) for r in rows]
probs = [float(r["smartturn"]) for r in rows]
states = [1.0 if r["state"] == "END" else 0.0 for r in rows]
fig, axes = plt.subplots(2, 1, figsize=(12, 5), sharex=True)
axes[0].plot(times, probs, color="#2563eb", marker="o", markersize=3, linewidth=1.2)
axes[0].axhline(0.5, color="#d62828", linestyle="--", linewidth=1.0)
axes[0].set_ylim(0, 1)
axes[0].set_ylabel("SmartTurn")
axes[0].grid(True, alpha=0.25)
axes[1].step(times, states, where="post", color="#2a9d8f", linewidth=1.4)
axes[1].set_ylim(-0.1, 1.1)
axes[1].set_yticks([0, 1], ["RUN", "END"])
axes[1].set_xlabel("Time (sec)")
axes[1].grid(True, alpha=0.25)
axes[1].set_xlim(0, max(duration_sec, 0.24))
fig.tight_layout()
fig.savefig(out_path, dpi=140)
plt.close(fig)
return str(out_path)
def run_smartturn_only_gradio(
enroll_audio: str,
mic_audio: str,
pvad_model_path: str,
pvad_threshold: float,
smartturn_model_selection: str,
smartturn_custom_model_path: str,
threshold: float,
pvad_silence_ms: float,
min_active_target_ms: float,
) -> tuple[str, str, str, str, Any, str | None]:
if not enroll_audio:
raise gr.Error("Upload an enrollment audio file.")
if not mic_audio:
raise gr.Error("Upload a mic audio file.")
smartturn_model_path = resolve_smartturn_model_path(smartturn_model_selection, smartturn_custom_model_path)
enroll = load_wav_16k(enroll_audio)
mic = load_wav_16k(mic_audio)
pvad = PvadOnnx(pvad_model_path, DEFAULT_SILERO_JIT)
smartturn = SmartTurnOnnx(smartturn_model_path)
pvad_probs = pvad.predict_target_probs(mic, enroll)
target_audio = make_pvad_target_audio(mic, pvad_probs, float(pvad_threshold))
frame_samples = 512
frame_ms = frame_samples / 16000.0 * 1000.0
silence_frames_required = 0 if pvad_silence_ms <= 0 else max(1, int(np.ceil(float(pvad_silence_ms) / frame_ms)))
total_frames = max(1, int(np.ceil(len(mic) / frame_samples)))
min_active_sec = float(min_active_target_ms) / 1000.0
threshold = float(threshold)
activity_mask = np.zeros_like(target_audio, dtype=np.float32)
rows: list[dict[str, Any]] = []
cuts: list[dict[str, Any]] = []
turn_start: int | None = None
silence_frames = 0
checked_current_silence = False
for idx in range(total_frames):
start = idx * frame_samples
end = min(len(mic), start + frame_samples)
if end <= start:
break
time_sec = start / 16000.0
active = frame_is_active(target_audio[start:end], threshold=0.01)
if active:
activity_mask[start:end] = 1.0
if active and turn_start is None:
turn_start = start
silence_frames = 0
checked_current_silence = False
elif active:
silence_frames = 0
checked_current_silence = False
elif turn_start is not None:
silence_frames += 1
if turn_start is None:
rows.append(
{
"time_sec": round(time_sec, 3),
"state": "IDLE",
"source": "idle",
"smartturn": 0.0,
"target_active_sec": 0.0,
"target_samples": 0,
}
)
continue
silence_due = silence_frames >= silence_frames_required and not checked_current_silence
if not silence_due:
continue
buffer = target_audio[turn_start:end].copy() * activity_mask[turn_start:end]
prob = smartturn.predict_prob(buffer)
active_samples = int(np.count_nonzero(np.abs(buffer) > 1e-8))
active_sec = active_samples / 16000.0
state = "END" if active_samples > 0 and prob > threshold and active_sec >= min_active_sec else "RUN"
source = "pvad_silence_smartturn" if state == "END" else "pvad_silence_incomplete"
rows.append(
{
"time_sec": round(time_sec, 3),
"state": state,
"source": source,
"smartturn": round(prob, 4),
"pvad_silence_sec": round(silence_frames * frame_samples / 16000.0, 3),
"target_active_sec": round(active_sec, 3),
"target_samples": active_samples,
"turn_start_sec": round(turn_start / 16000.0, 3),
}
)
checked_current_silence = True
if state == "END":
cuts.append(
{
"start": turn_start / 16000.0,
"end": end / 16000.0,
"smartturn_end": end / 16000.0,
"duration": (end - turn_start) / 16000.0,
"smartturn": prob,
"note": "hard_end",
"audio": target_audio[turn_start:end].copy(),
}
)
turn_start = None
silence_frames = 0
checked_current_silence = False
if turn_start is not None and turn_start < len(target_audio):
end = len(target_audio)
buffer = target_audio[turn_start:end].copy() * activity_mask[turn_start:end]
active_samples = int(np.count_nonzero(np.abs(buffer) > 1e-8))
active_sec = active_samples / 16000.0
if active_samples > 0:
prob = smartturn.predict_prob(buffer)
state = "END" if prob > threshold and active_sec >= min_active_sec else "FLUSH"
rows.append(
{
"time_sec": round(end / 16000.0, 3),
"state": state,
"source": "final_flush",
"smartturn": round(prob, 4),
"pvad_silence_sec": round(silence_frames * frame_samples / 16000.0, 3),
"target_active_sec": round(active_sec, 3),
"target_samples": active_samples,
"turn_start_sec": round(turn_start / 16000.0, 3),
}
)
cuts.append(
{
"start": turn_start / 16000.0,
"end": end / 16000.0,
"smartturn_end": end / 16000.0,
"duration": (end - turn_start) / 16000.0,
"smartturn": prob,
"note": "final_flush",
"audio": target_audio[turn_start:end].copy(),
}
)
run_dir = RUNS / "smartturn_only"
run_dir.mkdir(parents=True, exist_ok=True)
timeline = plot_smartturn_only(rows, len(mic) / 16000.0, run_dir / "timeline.png")
target_wav = write_wav(run_dir / "pvad_target_audio.wav", target_audio)
cut_paths = []
cut_choices = []
cut_dir = run_dir / "turn_cuts"
for idx, cut in enumerate(cuts, start=1):
path = cut_dir / f"{idx:03d}_{cut['start']:.2f}_{cut['end']:.2f}.wav"
write_wav(path, cut["audio"])
path_str = str(path)
cut_paths.append(path_str)
cut_choices.append(
(
f"{idx:03d} | {cut['start']:.2f}s-{cut['end']:.2f}s | "
f"{cut['duration']:.2f}s | end={cut['smartturn_end']:.2f}s | smart={cut['smartturn']:.3f} | {cut.get('note', 'cut')}",
path_str,
)
)
first_cut = cut_paths[0] if cut_paths else None
first_end_time = next((float(r["time_sec"]) for r in rows if r["state"] == "END"), None)
has_final_flush = any(cut.get("note") == "final_flush" for cut in cuts)
summary = {
"state": "END_FOUND" if first_end_time is not None else ("FINAL_FLUSH" if has_final_flush else "NO_END"),
"num_cuts": len(cuts),
"turn_cuts": cut_paths,
"audio_duration_sec": round(len(mic) / 16000.0, 3),
"first_end_time_sec": None if first_end_time is None else round(first_end_time, 3),
"threshold": threshold,
"pvad_silence_before_smartturn_ms": float(pvad_silence_ms),
"pvad_silence_frames_required": silence_frames_required,
"pvad_frame_ms": 32.0,
"min_active_target_ms": float(min_active_target_ms),
"pvad_threshold": float(pvad_threshold),
"pvad_model": str(pvad_model_path),
"silero": str(DEFAULT_SILERO_JIT),
"smartturn": str(smartturn_model_path),
"rule": "PVAD-gate mic audio first. Run SmartTurn only after PVAD target silence. No DualTurn.",
}
return timeline, json.dumps(summary, indent=2), json.dumps(rows, indent=2), target_wav, gr.update(choices=cut_choices, value=first_cut), first_cut
def build_app() -> gr.Blocks:
sota_ck150 = prefer_existing(LOCAL_SOTA_PREVBEST_CK150_INT8, REMOTE_SOTA_PREVBEST_CK150_INT8)
sota_ck50 = prefer_existing(LOCAL_SOTA_PREVBEST_CK50_INT8, REMOTE_SOTA_PREVBEST_CK50_INT8)
sota_ck100 = prefer_existing(LOCAL_SOTA_PREVBEST_CK100_INT8, REMOTE_SOTA_PREVBEST_CK100_INT8)
sota_hardneg4k_ck50 = prefer_existing(LOCAL_SOTA_HARDNEG4K_CK50_INT8, REMOTE_SOTA_HARDNEG4K_CK50_INT8)
smartturn_choices = [
("base pretrained smartturn-v3.1 int8", str(DEFAULT_SMARTTURN_ONNX)),
("top1 F1/FNR prev_best_1.1 ck150 | F1 0.8558 FPR 0.1822 FNR 0.1142", str(sota_ck150)),
("top2 F1 prev_best_1.1 ck50 | F1 0.8555 FPR 0.1802 FNR 0.1162", str(sota_ck50)),
("top1 FPR hardneg4k ck50 | F1 0.8150 FPR 0.1663 FNR 0.1964", str(sota_hardneg4k_ck50)),
("top2 FPR prev_best_1.1 ck100 | F1 0.8471 FPR 0.1762 FNR 0.1343", str(sota_ck100)),
("top1 FNR prev_best_1.1 ck150 | F1 0.8558 FPR 0.1822 FNR 0.1142", str(sota_ck150)),
("top2 FNR prev_best_1.1 ck50 | F1 0.8555 FPR 0.1802 FNR 0.1162", str(sota_ck50)),
("custom", "custom"),
]
with gr.Blocks(title="Half Duuplex Demo") as demo:
gr.Markdown("## Half Duuplex Demo")
with gr.Tabs():
with gr.Tab("SmartTurn + DualTurn"):
with gr.Row():
enroll = gr.Audio(value=default_audio_value(DEFAULT_ENROLL), label="Enrollment", type="filepath")
mic = gr.Audio(value=default_audio_value(DEFAULT_MIC), label="Mic", type="filepath")
assistant = gr.Audio(value=default_audio_value(DEFAULT_ASSISTANT), label="Assistant echo", type="filepath")
with gr.Row():
mode = gr.Radio(["pvad_gated", "raw"], value="pvad_gated", label="Audio mode")
device = gr.Radio(["cuda", "cpu"], value=DEFAULT_DEVICE, label="DualTurn device")
append_assistant = gr.Checkbox(value=True, label="Append assistant after END")
with gr.Row():
smart_threshold = gr.Slider(0, 1, value=0.5, step=0.01, label="SmartTurn END threshold")
pvad_threshold = gr.Slider(0, 1, value=0.5, step=0.01, label="PVAD target threshold")
pvad_model = gr.Dropdown(
choices=[
("h64 pvad_core.onnx", str(DEFAULT_PVAD_ONNX)),
("h256 pvad_core_h256.onnx", str(DEFAULT_PVAD_H256_ONNX)),
],
value=str(DEFAULT_PVAD_ONNX),
label="PVAD ONNX model",
)
smartturn_model = gr.Dropdown(
choices=smartturn_choices,
value=str(DEFAULT_SMARTTURN_ONNX),
label="SmartTurn ONNX model",
)
smartturn_custom_model = gr.Textbox(
value="",
label="Custom SmartTurn ONNX path",
placeholder="/path/to/model.onnx",
)
with gr.Row():
min_active = gr.Slider(0, 2000, value=300, step=50, label="Min active target ms")
check_ms = gr.Slider(80, 1000, value=240, step=80, label="Model check interval ms")
silence_fallback = gr.Slider(0, 3000, value=800, step=100, label="Silence fallback END ms")
with gr.Row():
asr_cut_silence = gr.Slider(0, 3000, value=2000, step=100, label="ASR cut silence ms")
assistant_max = gr.Slider(0.5, 12, value=3, step=0.5, label="Assistant max playback sec")
run = gr.Button("Run", variant="primary")
timeline = gr.Image(label="Timeline", type="filepath")
with gr.Row():
target_audio = gr.Audio(label="PVAD target audio", type="filepath")
mic_assistant_audio = gr.Audio(label="Mic with assistant echo", type="filepath")
with gr.Row():
cut_selector = gr.Dropdown(label="Model / ASR input audio cuts", choices=[], value=None)
cut_audio = gr.Audio(label="Selected cut audio", type="filepath")
summary = gr.Code(label="Summary JSON", language="json")
rows = gr.Code(label="State transitions", language="json")
run.click(
run_gradio,
inputs=[
enroll,
mic,
assistant,
mode,
smart_threshold,
pvad_threshold,
pvad_model,
smartturn_model,
smartturn_custom_model,
min_active,
silence_fallback,
asr_cut_silence,
check_ms,
append_assistant,
assistant_max,
device,
],
outputs=[timeline, summary, rows, target_audio, mic_assistant_audio, cut_selector, cut_audio],
)
cut_selector.change(select_cut_audio, inputs=[cut_selector], outputs=[cut_audio])
with gr.Tab("SmartTurn Only"):
with gr.Row():
st_enroll = gr.Audio(value=default_audio_value(DEFAULT_ENROLL), label="Enrollment", type="filepath")
st_mic = gr.Audio(value=default_audio_value(DEFAULT_MIC), label="Mic", type="filepath")
with gr.Row():
st_pvad_model = gr.Dropdown(
choices=[
("h64 pvad_core.onnx", str(DEFAULT_PVAD_ONNX)),
("h256 pvad_core_h256.onnx", str(DEFAULT_PVAD_H256_ONNX)),
],
value=str(DEFAULT_PVAD_ONNX),
label="PVAD ONNX model",
)
st_pvad_threshold = gr.Slider(0, 1, value=0.5, step=0.01, label="PVAD target threshold")
st_smartturn_model = gr.Dropdown(
choices=smartturn_choices,
value=str(DEFAULT_SMARTTURN_ONNX),
label="SmartTurn ONNX model",
)
st_smartturn_custom_model = gr.Textbox(
value="",
label="Custom SmartTurn ONNX path",
placeholder="/path/to/model.onnx",
)
with gr.Row():
st_threshold = gr.Slider(0, 1, value=0.5, step=0.01, label="SmartTurn END threshold")
st_pvad_silence = gr.Slider(0, 1000, value=200, step=20, label="PVAD silence before SmartTurn ms")
st_min_active = gr.Slider(0, 2000, value=300, step=50, label="Min active target ms")
st_run = gr.Button("Run SmartTurn Only", variant="primary")
st_timeline = gr.Image(label="Timeline", type="filepath")
st_target_audio = gr.Audio(label="PVAD target audio passed to SmartTurn", type="filepath")
with gr.Row():
st_cut_selector = gr.Dropdown(label="SmartTurn END cuts", choices=[], value=None)
st_cut_audio = gr.Audio(label="Selected cut audio", type="filepath")
st_summary = gr.Code(label="Summary JSON", language="json")
st_rows = gr.Code(label="Checks", language="json")
st_run.click(
run_smartturn_only_gradio,
inputs=[
st_enroll,
st_mic,
st_pvad_model,
st_pvad_threshold,
st_smartturn_model,
st_smartturn_custom_model,
st_threshold,
st_pvad_silence,
st_min_active,
],
outputs=[st_timeline, st_summary, st_rows, st_target_audio, st_cut_selector, st_cut_audio],
)
st_cut_selector.change(select_cut_audio, inputs=[st_cut_selector], outputs=[st_cut_audio])
return demo
if __name__ == "__main__":
server_port = int(os.environ.get("PORT", os.environ.get("GRADIO_SERVER_PORT", "7860")))
build_app().queue().launch(server_name="0.0.0.0", server_port=server_port)