viks66 commited on
Commit
56afddc
·
verified ·
1 Parent(s): 799623f

remove turn-bench-submission/predict.py

Browse files
Files changed (1) hide show
  1. turn-bench-submission/predict.py +0 -169
turn-bench-submission/predict.py DELETED
@@ -1,169 +0,0 @@
1
- # Author: Sathvik Udupa (2026)
2
- # Email: udupa@fit.vutbr.cz
3
- # Paper: Streaming Endpointer for Spoken Dialogue using Neural Audio Codecs and Label-Delayed Training, https://arxiv.org/abs/2506.07081, ASRU 2025
4
-
5
- """Mimi Endpointer — DiscriminativeModel for the TURN benchmark.
6
-
7
- Two-stream LSTM over Mimi embeddings, streamed 20ms chunk at a time.
8
- Mimi operates in 1920-sample (80ms) chunks → 2 LSTM frames per chunk.
9
- Four harness steps are buffered before each Mimi run; floor bit is held
10
- between updates. Inherent latency: ~80ms.
11
-
12
- floor = 1 if P(user) > threshold else 0
13
-
14
- subject is always fed as channel 0 (user); other as channel 1 (system).
15
-
16
- Debug mode (MIMI_DEBUG=1): saves debug_pass{N}.npz per conversation pass;
17
- run plot_debug.py afterwards to render PNGs.
18
-
19
- Sweep mode (MIMI_SWEEP=1): runs the harness sweep over thresholds 0.05–0.95
20
- in a single inference pass; threshold is reported per-step as a list[int].
21
- """
22
- from __future__ import annotations
23
-
24
- import atexit
25
- import os
26
- import sys
27
- from pathlib import Path
28
-
29
- import numpy as np
30
- import torch
31
-
32
- _HERE = Path(__file__).resolve().parent
33
- sys.path.insert(0, str(_HERE.parent)) # expose model.py from baselines/mimi_endpointer/
34
-
35
- from model import ( # noqa: E402
36
- AudioFeatureExtractor,
37
- AUDIO_DEFAULTS,
38
- load_model as load_mimi_model,
39
- )
40
-
41
- IDX_USER = 4 # from training config: {bos:0, system_end:1, user_end:2, system:3, user:4}
42
-
43
- CHECKPOINT = _HERE.parent / "checkpoint.pt"
44
-
45
- _CHUNK_STEPS = 4 # 4 × 20ms = 80ms = one Mimi frame_size (1920 samples at 24kHz)
46
- _SR = 24_000
47
- _FRAME_RATE = 50 # harness step rate (Hz)
48
-
49
-
50
- class MimiEndpointerModel:
51
- input_sample_rate = _SR # Mimi native rate; 24000 % 50 == 0 → 480 samples/step
52
-
53
- def __init__(
54
- self,
55
- threshold: float = 0.5,
56
- thresholds: list[float] | None = None,
57
- debug: bool = False,
58
- ) -> None:
59
- # sweep mode: thresholds is a list; single mode: scalar threshold
60
- if thresholds is not None:
61
- self.thresholds = thresholds # harness detects sweep mode via hasattr
62
- self._thresholds_arr = thresholds
63
- else:
64
- self.threshold = threshold # single operating point
65
- self._thresholds_arr = [threshold]
66
- self._sweep = thresholds is not None
67
- self.debug = debug
68
- device = "cuda" if torch.cuda.is_available() else "cpu"
69
- self._device = device
70
- self._model = load_mimi_model(str(CHECKPOINT), device=device)
71
- self._extractor = AudioFeatureExtractor(**AUDIO_DEFAULTS, device=device)
72
- self._ctx = None
73
- self._debug_idx = 0
74
- self._log_subj: list[np.ndarray] = []
75
- self._log_other: list[np.ndarray] = []
76
- self._log_floor: list[int] = []
77
- self._log_probs: list[np.ndarray] = [] # all 5 class probs per step (T, 5)
78
- if debug:
79
- atexit.register(self._save_npz)
80
- self.reset()
81
-
82
- def reset(self) -> None:
83
- if self.debug:
84
- self._save_npz()
85
- if self._ctx is not None:
86
- self._ctx.__exit__(None, None, None)
87
- self._ctx = self._extractor.mimi.streaming(batch_size=2)
88
- self._ctx.__enter__()
89
- self._h1, self._c1 = self._model.init_hidden(1, self._device)
90
- self._h2, self._c2 = self._model.init_hidden(1, self._device)
91
- self._buf_subj: list[np.ndarray] = []
92
- self._buf_other: list[np.ndarray] = []
93
- self._floor_bits: list[int] = [0] * len(self._thresholds_arr)
94
- self._log_subj = []
95
- self._log_other = []
96
- self._log_floor = []
97
- self._log_probs = []
98
-
99
- def __del__(self) -> None:
100
- if self._ctx is not None:
101
- self._ctx.__exit__(None, None, None)
102
-
103
- def _save_npz(self) -> None:
104
- if not self._log_floor:
105
- return
106
- out = _HERE / f"debug_pass{self._debug_idx}.npz"
107
- np.savez(
108
- out,
109
- subj=np.concatenate(self._log_subj),
110
- other=np.concatenate(self._log_other),
111
- floor=np.array(self._log_floor, dtype=np.int8),
112
- probs=np.array(self._log_probs, dtype=np.float32), # (T, 5)
113
- threshold=np.float32(self._thresholds_arr[0]),
114
- sr=np.int32(_SR),
115
- frame_rate=np.int32(_FRAME_RATE),
116
- )
117
- sys.stderr.write(f"[debug] saved → {out}\n")
118
- self._debug_idx += 1
119
-
120
- def step(self, subject_audio: np.ndarray, other_audio: np.ndarray):
121
- self._buf_subj.append(subject_audio)
122
- self._buf_other.append(other_audio)
123
-
124
- if self.debug:
125
- self._log_subj.append(subject_audio)
126
- self._log_other.append(other_audio)
127
-
128
- new_probs: np.ndarray | None = None
129
-
130
- if len(self._buf_subj) == _CHUNK_STEPS:
131
- chunk_s = torch.from_numpy(np.concatenate(self._buf_subj)).to(self._device)
132
- chunk_o = torch.from_numpy(np.concatenate(self._buf_other)).to(self._device)
133
- self._buf_subj.clear()
134
- self._buf_other.clear()
135
-
136
- # (2, 1, 1920) — subject=channel 0 (user), other=channel 1 (system)
137
- chunk = torch.stack([chunk_s, chunk_o]).unsqueeze(1)
138
- with torch.no_grad():
139
- emb = self._extractor.mimi.encode_to_latent(chunk, quantize=True) # (2, feat, 1)
140
- emb = self._extractor.mimi.upsample(emb) # (2, feat, 2)
141
- logits = None
142
- for t in range(emb.shape[-1]):
143
- logits, self._h1, self._c1, self._h2, self._c2 = \
144
- self._model.infer_ar_step(
145
- emb[0:1, :, t], emb[1:2, :, t],
146
- self._h1, self._c1, self._h2, self._c2,
147
- )
148
- new_probs = torch.softmax(logits[0], dim=-1).cpu().numpy() # (5,)
149
- p_user = new_probs[IDX_USER]
150
- self._floor_bits = [1 if p_user > t else 0 for t in self._thresholds_arr]
151
-
152
- if self.debug:
153
- p = new_probs if new_probs is not None else (
154
- self._log_probs[-1] if self._log_probs else np.zeros(5, dtype=np.float32)
155
- )
156
- self._log_probs.append(p)
157
- self._log_floor.append(self._floor_bits[0]) # first threshold for debug plot
158
-
159
- return self._floor_bits if self._sweep else self._floor_bits[0]
160
-
161
-
162
- def load_model() -> MimiEndpointerModel:
163
- debug = os.environ.get("MIMI_DEBUG", "0") == "1"
164
- sweep = os.environ.get("MIMI_SWEEP", "0") == "1"
165
- if sweep:
166
- thresholds = list(np.round(np.arange(0.05, 1.0, 0.05), 2).tolist())
167
- return MimiEndpointerModel(thresholds=thresholds, debug=debug)
168
- thr = float(os.environ.get("MIMI_THRESHOLD", "0.1"))
169
- return MimiEndpointerModel(threshold=thr, debug=debug)