svec commited on
Commit
0e99521
Β·
verified Β·
1 Parent(s): 3d77c9d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +247 -60
app.py CHANGED
@@ -5,6 +5,7 @@ import time
5
  import datetime
6
  import threading
7
  import concurrent.futures
 
8
  import gradio as gr
9
  import numpy as np
10
  import pandas as pd
@@ -23,6 +24,11 @@ EMBED_THREADS = 2 # per-worker thread count β€” NUMA sweet spot
23
  API_TIMEOUT = 12
24
  STREAMING_TIMEOUT = 120
25
 
 
 
 
 
 
26
  # Set thread count before model load so ORT/MKL picks it up
27
  torch.set_num_threads(EMBED_THREADS)
28
 
@@ -30,9 +36,8 @@ torch.set_num_threads(EMBED_THREADS)
30
  def _ts() -> str:
31
  return datetime.datetime.now().strftime("%H:%M:%S.%f")[:-3]
32
 
 
33
  DATASETS_SERVER = "https://datasets-server.huggingface.co"
34
- MODEL_ID = "microsoft/wavlm-base-plus-sv"
35
- TARGET_SR = 16000
36
 
37
  _feature_extractor = None
38
  _model = None
@@ -48,34 +53,115 @@ def _load_model():
48
  return _feature_extractor, _model
49
 
50
 
51
- def _to_array(audio_array: np.ndarray, sr: int, max_sec: int) -> np.ndarray:
52
- waveform = torch.tensor(audio_array, dtype=torch.float32)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
53
  if waveform.ndim == 2:
54
- waveform = waveform.mean(0)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
  if sr != TARGET_SR:
56
- waveform = torchaudio.functional.resample(waveform, sr, TARGET_SR)
57
- return waveform[: max_sec * TARGET_SR].numpy().astype(np.float32)
 
 
 
 
 
 
 
 
 
58
 
59
 
60
  def _embed_one(array: np.ndarray) -> np.ndarray:
61
  """Embed a single clip. Thread-safe: eval()+no_grad(), GIL released in C++."""
62
  fe, mdl = _load_model()
 
 
 
 
 
 
 
 
63
  inputs = fe(array, sampling_rate=TARGET_SR, return_tensors="pt")
64
  with torch.no_grad():
65
  out = mdl(**inputs)
66
- return out.embeddings.squeeze().numpy()
67
 
68
 
69
  def _parallel_embed(arrays: list[np.ndarray], log: list[str]) -> np.ndarray:
70
  """Embed all clips using N_WORKERS parallel threads (threads=2 each)."""
 
 
 
71
  t0 = time.time()
72
- log.append(f"[{_ts()}] --- Phase 2: embed {len(arrays)} clips "
73
- f"({N_WORKERS} workers Γ— {EMBED_THREADS} threads) ---")
 
 
 
74
  with concurrent.futures.ThreadPoolExecutor(max_workers=N_WORKERS) as ex:
75
  futures = [ex.submit(_embed_one, arr) for arr in arrays]
76
  results = [f.result() for f in futures]
 
77
  ms = int((time.time() - t0) * 1000)
78
- log.append(f"[{_ts()}] embed done: {ms}ms total, {ms//len(arrays)}ms/clip avg")
 
79
  return np.stack(results)
80
 
81
 
@@ -84,7 +170,7 @@ def _parallel_embed(arrays: list[np.ndarray], log: list[str]) -> np.ndarray:
84
  def _parse_audio_urls(rows: list) -> list[str]:
85
  urls = []
86
  for row in rows:
87
- audio = row["row"].get("audio", {})
88
  if isinstance(audio, list):
89
  audio = audio[0] if audio else {}
90
  if isinstance(audio, dict) and "src" in audio:
@@ -96,17 +182,25 @@ def _try_endpoint(endpoint: str, params: dict, headers: dict) -> tuple[list[str]
96
  """Single request attempt. Returns (urls, status_str)."""
97
  try:
98
  t0 = time.time()
99
- resp = requests.get(f"{DATASETS_SERVER}{endpoint}", params=params,
100
- headers=headers, timeout=API_TIMEOUT)
 
 
 
 
101
  elapsed = int((time.time() - t0) * 1000)
 
102
  if not resp.ok:
103
  return [], f"{endpoint} HTTP {resp.status_code} ({elapsed}ms)"
 
104
  rows = resp.json().get("rows", [])
105
  if not rows:
106
  return [], f"{endpoint} empty ({elapsed}ms)"
 
107
  urls = _parse_audio_urls(rows)
108
  if urls:
109
  return urls, f"{endpoint} {elapsed}ms"
 
110
  return [], f"{endpoint} no src ({elapsed}ms)"
111
  except requests.Timeout:
112
  return [], f"{endpoint} timeout>{API_TIMEOUT}s"
@@ -116,15 +210,35 @@ def _try_endpoint(endpoint: str, params: dict, headers: dict) -> tuple[list[str]
116
 
117
  def _fetch_audio_urls(repo_id: str, n: int, token: str | None) -> tuple[list[str], str]:
118
  """Fire /rows and /first-rows in parallel, return first successful result.
119
- Uses shutdown(wait=False) so the losing request doesn't block the caller."""
 
 
120
  headers = {"Authorization": f"Bearer {token}"} if token else {}
121
  calls = [
122
- ("/rows", {"dataset": repo_id, "config": "default", "split": "train", "offset": 0, "length": n}),
123
- ("/first-rows", {"dataset": repo_id, "config": "default", "split": "train"}),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
124
  ]
 
125
  ex = concurrent.futures.ThreadPoolExecutor(max_workers=2)
126
  futs = {ex.submit(_try_endpoint, ep, params, headers): ep for ep, params in calls}
127
  errs = []
 
128
  try:
129
  for fut in concurrent.futures.as_completed(futs, timeout=API_TIMEOUT + 2):
130
  urls, status = fut.result()
@@ -136,6 +250,7 @@ def _fetch_audio_urls(repo_id: str, n: int, token: str | None) -> tuple[list[str
136
  errs.append("both endpoints timed out")
137
  finally:
138
  ex.shutdown(wait=False, cancel_futures=True)
 
139
  return [], " | ".join(errs)
140
 
141
 
@@ -143,15 +258,22 @@ def _download_audio(url: str, token: str | None, max_sec: int) -> tuple[np.ndarr
143
  """Download one audio file. Returns (array, size_kb, dl_ms)."""
144
  headers = {"Authorization": f"Bearer {token}"} if token else {}
145
  t0 = time.time()
 
146
  resp = requests.get(url, headers=headers, timeout=30)
147
  resp.raise_for_status()
 
148
  dl_ms = int((time.time() - t0) * 1000)
149
- audio_array, sr = sf.read(io.BytesIO(resp.content))
150
- return _to_array(audio_array, sr, max_sec), len(resp.content) // 1024, dl_ms
 
 
151
 
152
 
153
  def _fetch_repo_audio(
154
- repo: str, n_samples: int, audio_sec: int, token: str | None,
 
 
 
155
  log: list[str],
156
  ) -> tuple[str, list[np.ndarray]]:
157
  """Fetch + download audio for one repo. Appends timestamped entries to log."""
@@ -160,44 +282,62 @@ def _fetch_repo_audio(
160
 
161
  try:
162
  log.append(f"[{_ts()}] {short}: fetching URLs from datasets-server…")
163
- urls, api_status = _fetch_audio_urls(repo, n_samples, token)
164
- fetch_ms = int((time.time() - t0) * 1000)
165
 
166
  if urls:
167
  log.append(f"[{_ts()}] {short}: {api_status} β†’ {len(urls)} URLs")
168
- with concurrent.futures.ThreadPoolExecutor(max_workers=len(urls)) as ex:
169
  futures = [ex.submit(_download_audio, u, token, audio_sec) for u in urls]
170
  results = [f.result() for f in futures]
171
- arrays = [r[0] for r in results]
172
- sizes = [r[1] for r in results]
173
- dls = [r[2] for r in results]
 
174
  total_ms = int((time.time() - t0) * 1000)
 
175
  log.append(
176
  f"[{_ts()}] {short}: βœ“ {len(arrays)} clips "
177
- f"dl=[{', '.join(str(d)+'ms' for d in dls)}] "
178
- f"size=[{', '.join(str(s)+'KB' for s in sizes)}] "
179
  f"total={total_ms}ms"
180
  )
181
  return repo, arrays
182
 
183
  # Streaming fallback with timeout
184
- log.append(f"[{_ts()}] {short}: ⚠ datasets-server failed ({api_status}) β†’ streaming fallback (timeout={STREAMING_TIMEOUT}s)")
 
 
 
185
  t1 = time.time()
186
 
187
  def _stream() -> list[np.ndarray]:
188
- from datasets import load_dataset, Audio as HFAudio
 
 
189
  ds = load_dataset(repo, split="train", streaming=True, token=token)
190
  ds = ds.cast_column("audio", HFAudio(decode=False))
 
191
  result = []
192
  for j, row in enumerate(ds):
193
- if j >= n_samples:
194
  break
 
195
  raw = row["audio"]
196
- audio_bytes = raw.get("bytes") or open(raw["path"], "rb").read()
197
- a, sr = sf.read(io.BytesIO(audio_bytes))
 
 
 
 
 
198
  result.append(_to_array(a, sr, audio_sec))
 
199
  elapsed = int((time.time() - t1) * 1000)
200
- log.append(f"[{_ts()}] {short}: streaming clip {j+1}/{n_samples} ({len(audio_bytes)//1024}KB, {elapsed}ms so far)")
 
 
 
 
201
  return result
202
 
203
  with concurrent.futures.ThreadPoolExecutor(max_workers=1) as ex:
@@ -205,15 +345,21 @@ def _fetch_repo_audio(
205
  try:
206
  arrays = fut.result(timeout=STREAMING_TIMEOUT)
207
  total_ms = int((time.time() - t0) * 1000)
208
- log.append(f"[{_ts()}] {short}: βœ“ streaming done, {len(arrays)} clips, total={total_ms}ms")
 
 
 
209
  return repo, arrays
210
  except concurrent.futures.TimeoutError:
211
  total_ms = int((time.time() - t0) * 1000)
212
- log.append(f"[{_ts()}] {short}: βœ— streaming timeout after {STREAMING_TIMEOUT}s β€” skipping")
 
 
 
213
  return repo, []
214
 
215
  except Exception as exc:
216
- log.append(f"[{_ts()}] {short}: βœ— {exc} (total={int((time.time()-t0)*1000)}ms)")
217
  return repo, []
218
 
219
 
@@ -236,16 +382,19 @@ def identify_speakers(
236
  log: list[str] = []
237
  t_total = time.time()
238
 
239
- log.append(f"[{_ts()}] === START: {len(repos)} repos, {samples_per_book} sample/book, {audio_sec}s/clip ===")
 
 
 
240
  log.append(f"[{_ts()}] CPU count: {N_CPUS}, workers: {N_WORKERS}, embed_threads: {EMBED_THREADS}")
241
 
242
  progress(0, desc="Loading model…")
243
  t_model = time.time()
244
  _load_model()
245
- log.append(f"[{_ts()}] model loaded in {int((time.time()-t_model)*1000)}ms")
246
 
247
  # ── Phase 1: download all audio in parallel (I/O bound) ──────────────────
248
- log.append(f"[{_ts()}] --- Phase 1: download ({N_CPUS*2} workers) ---")
249
  progress(0.05, desc=f"Downloading audio from {len(repos)} datasets…")
250
  repo_arrays: dict[str, list[np.ndarray]] = {}
251
 
@@ -258,13 +407,21 @@ def identify_speakers(
258
  for future in concurrent.futures.as_completed(futures):
259
  repo, arrays = future.result()
260
  done += 1
261
- progress(0.05 + 0.55 * done / len(repos), desc=f"[{done}/{len(repos)}] {repo.split('/')[-1]}")
 
 
 
262
  if arrays:
263
  repo_arrays[repo] = arrays
 
 
264
 
265
  ok = len(repo_arrays)
266
  failed = len(repos) - ok
267
- log.append(f"[{_ts()}] Phase 1 done: {ok} ok, {failed} failed, phase_total={int((time.time()-t_total)*1000)}ms")
 
 
 
268
 
269
  if not repo_arrays:
270
  return pd.DataFrame(), "No audio downloaded.", "\n".join(log), "", None
@@ -280,18 +437,31 @@ def identify_speakers(
280
  all_arrays.extend(repo_arrays[repo])
281
  repo_slices[repo] = (start, len(all_arrays))
282
 
283
- all_embeddings = _parallel_embed(all_arrays, log)
 
 
 
 
284
 
285
  # ── Phase 3: average per repo, cluster ───────────────────────────────────
286
  log.append(f"[{_ts()}] --- Phase 3: cluster ({len(repo_slices)} repos) ---")
287
  progress(0.95, desc="Clustering…")
 
288
  embeddings: dict[str, np.ndarray] = {}
289
  for repo, (start, end) in repo_slices.items():
290
- embeddings[repo] = all_embeddings[start:end].mean(axis=0)
 
 
 
 
291
 
292
  repo_names = list(embeddings.keys())
293
  emb_matrix = np.stack([embeddings[r] for r in repo_names])
294
- emb_matrix = emb_matrix / np.linalg.norm(emb_matrix, axis=1, keepdims=True)
 
 
 
 
295
  sim_matrix = np.clip(emb_matrix @ emb_matrix.T, -1.0, 1.0)
296
  dist_matrix = 1.0 - sim_matrix
297
  np.fill_diagonal(dist_matrix, 0.0)
@@ -310,32 +480,38 @@ def identify_speakers(
310
  rows = []
311
  for i, repo in enumerate(repo_names):
312
  cluster = labels[i]
313
- same_idx = [j for j, l in enumerate(labels) if l == cluster and j != i]
314
  intra_sim = float(np.mean([sim_matrix[i][j] for j in same_idx])) if same_idx else 1.0
 
315
  other_sorted = sorted([j for j in range(n) if j != i], key=lambda j: -sim_matrix[i][j])
316
  closest = (
317
  f"{repo_names[other_sorted[0]].split('/')[-1]} ({sim_matrix[i][other_sorted[0]]:.2f})"
318
- if other_sorted else "-"
 
 
 
 
 
 
 
 
 
 
 
319
  )
320
- rows.append({
321
- "dataset": repo.split("/")[-1],
322
- "speaker_id": f"speaker_{cluster + 1:02d}",
323
- "books_with_speaker": sum(1 for l in labels if l == cluster),
324
- "intra_sim": round(intra_sim, 3),
325
- "closest_match": closest,
326
- })
327
 
328
  df = pd.DataFrame(rows).sort_values(["speaker_id", "dataset"]).reset_index(drop=True)
329
  n_speakers = len(set(labels))
330
  total_ms = int((time.time() - t_total) * 1000)
331
- summary = f"βœ… {len(repo_names)} books β†’ {n_speakers} unique speakers ({total_ms/1000:.1f}s total)"
332
  log.append(f"[{_ts()}] === DONE: {total_ms}ms total ===")
333
 
334
  # Plain-text copy-friendly output (space-separated, matches log format)
335
  text_lines = []
336
  for _, r in df.iterrows():
337
  text_lines.append(
338
- f"{r['dataset']} {r['speaker_id']} {r['books_with_speaker']} {r['intra_sim']} {r['closest_match']}"
 
339
  )
340
  plain_text = "\n".join(text_lines)
341
 
@@ -352,7 +528,7 @@ DESCRIPTION = """
352
 
353
  Finds unique speakers across multiple HF audio datasets. Each dataset is assumed to have
354
  **one speaker** (e.g. an audiobook). The app fetches audio directly via the datasets-server API
355
- (no full parquet download), downloads in parallel, then embeds clips across 8 parallel workers.
356
 
357
  **Model:** [microsoft/wavlm-base-plus-sv](https://huggingface.co/microsoft/wavlm-base-plus-sv) β€”
358
  language-agnostic speaker embeddings, works for any language.
@@ -367,7 +543,7 @@ language-agnostic speaker embeddings, works for any language.
367
  - *Audio length (sec)* β€” seconds of each chunk to use for embedding. 5 sec is sufficient for a clear voice.
368
  - *Same-speaker threshold* β€” cosine similarity cutoff. Raise if too many books merge into one speaker; lower if one person gets split across IDs.
369
  - *HF Token* β€” only needed for **private** repos.
370
- 3. Click **Identify Speakers**. Downloads run in parallel, then all clips are embedded in one batch β€” expect ~20–40 sec for 35 books.
371
 
372
  ## Output columns
373
 
@@ -384,9 +560,11 @@ language-agnostic speaker embeddings, works for any language.
384
  The **Errors / Timing** box shows per-dataset timing and batch embed stats β€” useful for diagnosing slow datasets.
385
  """
386
 
 
387
  with gr.Blocks(title="Speaker Identifier") as demo:
388
  gr.Markdown(DESCRIPTION)
389
  gr.Markdown("---")
 
390
  with gr.Row():
391
  with gr.Column(scale=2):
392
  repo_input = gr.Textbox(
@@ -397,12 +575,18 @@ with gr.Blocks(title="Speaker Identifier") as demo:
397
  with gr.Column(scale=1):
398
  samples = gr.Slider(1, 10, value=3, step=1, label="Samples per book")
399
  audio_sec = gr.Slider(
400
- 2, 30, value=5, step=1,
 
 
 
401
  label="Audio length per sample (sec)",
402
  info="5 sec is usually enough; longer = more accurate but slower",
403
  )
404
  threshold = gr.Slider(
405
- 0.60, 0.98, value=0.82, step=0.01,
 
 
 
406
  label="Same-speaker threshold",
407
  info="Higher = stricter matching β†’ more clusters",
408
  )
@@ -419,6 +603,7 @@ with gr.Blocks(title="Speaker Identifier") as demo:
419
  headers=["dataset", "speaker_id", "books_with_speaker", "intra_sim", "closest_match"],
420
  wrap=True,
421
  )
 
422
  with gr.Row():
423
  text_out = gr.Textbox(
424
  label="Plain text (copy-friendly)",
@@ -427,6 +612,7 @@ with gr.Blocks(title="Speaker Identifier") as demo:
427
  info="dataset speaker_id n_books intra_sim closest_match",
428
  )
429
  csv_out = gr.File(label="Download CSV", file_types=[".csv"])
 
430
  errors_out = gr.Textbox(label="Errors / Timing", interactive=False)
431
 
432
  run_btn.click(
@@ -435,4 +621,5 @@ with gr.Blocks(title="Speaker Identifier") as demo:
435
  outputs=[table_out, summary_out, errors_out, text_out, csv_out],
436
  )
437
 
 
438
  demo.launch()
 
5
  import datetime
6
  import threading
7
  import concurrent.futures
8
+
9
  import gradio as gr
10
  import numpy as np
11
  import pandas as pd
 
24
  API_TIMEOUT = 12
25
  STREAMING_TIMEOUT = 120
26
 
27
+ MODEL_ID = "microsoft/wavlm-base-plus-sv"
28
+ TARGET_SR = 16000
29
+ MIN_AUDIO_SEC = 1 # WavLM must not receive empty/tiny clips
30
+ MIN_AUDIO_SAMPLES = TARGET_SR * MIN_AUDIO_SEC
31
+
32
  # Set thread count before model load so ORT/MKL picks it up
33
  torch.set_num_threads(EMBED_THREADS)
34
 
 
36
  def _ts() -> str:
37
  return datetime.datetime.now().strftime("%H:%M:%S.%f")[:-3]
38
 
39
+
40
  DATASETS_SERVER = "https://datasets-server.huggingface.co"
 
 
41
 
42
  _feature_extractor = None
43
  _model = None
 
53
  return _feature_extractor, _model
54
 
55
 
56
+ def _mono_1d_tensor(audio_array: np.ndarray) -> torch.Tensor:
57
+ """Convert audio from soundfile/torchaudio style arrays to mono 1-D float tensor.
58
+
59
+ soundfile normally returns:
60
+ - mono: shape (frames,)
61
+ - stereo: shape (frames, channels)
62
+
63
+ torchaudio often uses:
64
+ - shape (channels, frames)
65
+
66
+ The old bug was using mean(0) for soundfile stereo. For shape
67
+ (frames, channels), mean(0) collapses frames and leaves only 1-2 samples,
68
+ which later crashes WavLM conv1d with: kernel size > input size.
69
+ """
70
+ arr = np.asarray(audio_array)
71
+
72
+ if arr.size == 0:
73
+ return torch.zeros(0, dtype=torch.float32)
74
+
75
+ # Convert integers to float32 in a safe range when necessary.
76
+ if np.issubdtype(arr.dtype, np.integer):
77
+ info = np.iinfo(arr.dtype)
78
+ scale = float(max(abs(info.min), info.max))
79
+ arr = arr.astype(np.float32) / scale
80
+ else:
81
+ arr = arr.astype(np.float32, copy=False)
82
+
83
+ arr = np.nan_to_num(arr, nan=0.0, posinf=0.0, neginf=0.0)
84
+ waveform = torch.from_numpy(arr)
85
+
86
+ if waveform.ndim == 1:
87
+ return waveform.contiguous()
88
+
89
  if waveform.ndim == 2:
90
+ # Heuristic:
91
+ # - soundfile: (frames, channels), usually channels <= 8 and frames >> channels
92
+ # - torchaudio: (channels, frames), usually first dim <= 8
93
+ if waveform.shape[0] <= 8 and waveform.shape[1] > waveform.shape[0]:
94
+ # channels-first -> average channels
95
+ return waveform.mean(dim=0).contiguous()
96
+ # frames-first -> average channels; this is the important fix
97
+ return waveform.mean(dim=1).contiguous()
98
+
99
+ # Fallback for unusual shapes: preserve time axis as the longest axis and
100
+ # average everything else.
101
+ time_axis = int(np.argmax(waveform.shape))
102
+ waveform = waveform.movedim(time_axis, 0)
103
+ waveform = waveform.reshape(waveform.shape[0], -1).mean(dim=1)
104
+ return waveform.contiguous()
105
+
106
+
107
+ def _to_array(audio_array: np.ndarray, sr: int, max_sec: int) -> np.ndarray:
108
+ waveform = _mono_1d_tensor(audio_array)
109
+
110
+ if waveform.numel() == 0:
111
+ waveform = torch.zeros(MIN_AUDIO_SAMPLES, dtype=torch.float32)
112
+
113
+ if int(sr) <= 0:
114
+ sr = TARGET_SR
115
+
116
  if sr != TARGET_SR:
117
+ waveform = torchaudio.functional.resample(waveform, int(sr), TARGET_SR)
118
+
119
+ max_samples = max(MIN_AUDIO_SAMPLES, int(max_sec) * TARGET_SR)
120
+ waveform = waveform[:max_samples]
121
+
122
+ # WavLM cannot process empty/tiny clips. Pad to at least 1 second.
123
+ if waveform.numel() < MIN_AUDIO_SAMPLES:
124
+ pad = MIN_AUDIO_SAMPLES - waveform.numel()
125
+ waveform = torch.nn.functional.pad(waveform, (0, pad))
126
+
127
+ return waveform.numpy().astype(np.float32, copy=False)
128
 
129
 
130
  def _embed_one(array: np.ndarray) -> np.ndarray:
131
  """Embed a single clip. Thread-safe: eval()+no_grad(), GIL released in C++."""
132
  fe, mdl = _load_model()
133
+
134
+ array = np.asarray(array, dtype=np.float32).reshape(-1)
135
+ array = np.nan_to_num(array, nan=0.0, posinf=0.0, neginf=0.0)
136
+
137
+ # Last-resort guard in case a bad array bypassed _to_array().
138
+ if array.size < MIN_AUDIO_SAMPLES:
139
+ array = np.pad(array, (0, MIN_AUDIO_SAMPLES - array.size), mode="constant")
140
+
141
  inputs = fe(array, sampling_rate=TARGET_SR, return_tensors="pt")
142
  with torch.no_grad():
143
  out = mdl(**inputs)
144
+ return out.embeddings.squeeze().detach().cpu().numpy().astype(np.float32, copy=False)
145
 
146
 
147
  def _parallel_embed(arrays: list[np.ndarray], log: list[str]) -> np.ndarray:
148
  """Embed all clips using N_WORKERS parallel threads (threads=2 each)."""
149
+ if not arrays:
150
+ raise ValueError("No audio arrays to embed.")
151
+
152
  t0 = time.time()
153
+ log.append(
154
+ f"[{_ts()}] --- Phase 2: embed {len(arrays)} clips "
155
+ f"({N_WORKERS} workers Γ— {EMBED_THREADS} threads) ---"
156
+ )
157
+
158
  with concurrent.futures.ThreadPoolExecutor(max_workers=N_WORKERS) as ex:
159
  futures = [ex.submit(_embed_one, arr) for arr in arrays]
160
  results = [f.result() for f in futures]
161
+
162
  ms = int((time.time() - t0) * 1000)
163
+ avg = ms // max(1, len(arrays))
164
+ log.append(f"[{_ts()}] embed done: {ms}ms total, {avg}ms/clip avg")
165
  return np.stack(results)
166
 
167
 
 
170
  def _parse_audio_urls(rows: list) -> list[str]:
171
  urls = []
172
  for row in rows:
173
+ audio = row.get("row", {}).get("audio", {})
174
  if isinstance(audio, list):
175
  audio = audio[0] if audio else {}
176
  if isinstance(audio, dict) and "src" in audio:
 
182
  """Single request attempt. Returns (urls, status_str)."""
183
  try:
184
  t0 = time.time()
185
+ resp = requests.get(
186
+ f"{DATASETS_SERVER}{endpoint}",
187
+ params=params,
188
+ headers=headers,
189
+ timeout=API_TIMEOUT,
190
+ )
191
  elapsed = int((time.time() - t0) * 1000)
192
+
193
  if not resp.ok:
194
  return [], f"{endpoint} HTTP {resp.status_code} ({elapsed}ms)"
195
+
196
  rows = resp.json().get("rows", [])
197
  if not rows:
198
  return [], f"{endpoint} empty ({elapsed}ms)"
199
+
200
  urls = _parse_audio_urls(rows)
201
  if urls:
202
  return urls, f"{endpoint} {elapsed}ms"
203
+
204
  return [], f"{endpoint} no src ({elapsed}ms)"
205
  except requests.Timeout:
206
  return [], f"{endpoint} timeout>{API_TIMEOUT}s"
 
210
 
211
  def _fetch_audio_urls(repo_id: str, n: int, token: str | None) -> tuple[list[str], str]:
212
  """Fire /rows and /first-rows in parallel, return first successful result.
213
+
214
+ Uses shutdown(wait=False) so the losing request doesn't block the caller.
215
+ """
216
  headers = {"Authorization": f"Bearer {token}"} if token else {}
217
  calls = [
218
+ (
219
+ "/rows",
220
+ {
221
+ "dataset": repo_id,
222
+ "config": "default",
223
+ "split": "train",
224
+ "offset": 0,
225
+ "length": n,
226
+ },
227
+ ),
228
+ (
229
+ "/first-rows",
230
+ {
231
+ "dataset": repo_id,
232
+ "config": "default",
233
+ "split": "train",
234
+ },
235
+ ),
236
  ]
237
+
238
  ex = concurrent.futures.ThreadPoolExecutor(max_workers=2)
239
  futs = {ex.submit(_try_endpoint, ep, params, headers): ep for ep, params in calls}
240
  errs = []
241
+
242
  try:
243
  for fut in concurrent.futures.as_completed(futs, timeout=API_TIMEOUT + 2):
244
  urls, status = fut.result()
 
250
  errs.append("both endpoints timed out")
251
  finally:
252
  ex.shutdown(wait=False, cancel_futures=True)
253
+
254
  return [], " | ".join(errs)
255
 
256
 
 
258
  """Download one audio file. Returns (array, size_kb, dl_ms)."""
259
  headers = {"Authorization": f"Bearer {token}"} if token else {}
260
  t0 = time.time()
261
+
262
  resp = requests.get(url, headers=headers, timeout=30)
263
  resp.raise_for_status()
264
+
265
  dl_ms = int((time.time() - t0) * 1000)
266
+ audio_array, sr = sf.read(io.BytesIO(resp.content), always_2d=False)
267
+ array = _to_array(audio_array, sr, max_sec)
268
+
269
+ return array, len(resp.content) // 1024, dl_ms
270
 
271
 
272
  def _fetch_repo_audio(
273
+ repo: str,
274
+ n_samples: int,
275
+ audio_sec: int,
276
+ token: str | None,
277
  log: list[str],
278
  ) -> tuple[str, list[np.ndarray]]:
279
  """Fetch + download audio for one repo. Appends timestamped entries to log."""
 
282
 
283
  try:
284
  log.append(f"[{_ts()}] {short}: fetching URLs from datasets-server…")
285
+ urls, api_status = _fetch_audio_urls(repo, int(n_samples), token)
 
286
 
287
  if urls:
288
  log.append(f"[{_ts()}] {short}: {api_status} β†’ {len(urls)} URLs")
289
+ with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, len(urls))) as ex:
290
  futures = [ex.submit(_download_audio, u, token, audio_sec) for u in urls]
291
  results = [f.result() for f in futures]
292
+
293
+ arrays = [r[0] for r in results if r[0].size >= MIN_AUDIO_SAMPLES]
294
+ sizes = [r[1] for r in results]
295
+ dls = [r[2] for r in results]
296
  total_ms = int((time.time() - t0) * 1000)
297
+
298
  log.append(
299
  f"[{_ts()}] {short}: βœ“ {len(arrays)} clips "
300
+ f"dl=[{', '.join(str(d) + 'ms' for d in dls)}] "
301
+ f"size=[{', '.join(str(s) + 'KB' for s in sizes)}] "
302
  f"total={total_ms}ms"
303
  )
304
  return repo, arrays
305
 
306
  # Streaming fallback with timeout
307
+ log.append(
308
+ f"[{_ts()}] {short}: ⚠ datasets-server failed ({api_status}) "
309
+ f"β†’ streaming fallback (timeout={STREAMING_TIMEOUT}s)"
310
+ )
311
  t1 = time.time()
312
 
313
  def _stream() -> list[np.ndarray]:
314
+ from datasets import Audio as HFAudio
315
+ from datasets import load_dataset
316
+
317
  ds = load_dataset(repo, split="train", streaming=True, token=token)
318
  ds = ds.cast_column("audio", HFAudio(decode=False))
319
+
320
  result = []
321
  for j, row in enumerate(ds):
322
+ if j >= int(n_samples):
323
  break
324
+
325
  raw = row["audio"]
326
+ if raw.get("bytes") is not None:
327
+ audio_bytes = raw["bytes"]
328
+ else:
329
+ with open(raw["path"], "rb") as fh:
330
+ audio_bytes = fh.read()
331
+
332
+ a, sr = sf.read(io.BytesIO(audio_bytes), always_2d=False)
333
  result.append(_to_array(a, sr, audio_sec))
334
+
335
  elapsed = int((time.time() - t1) * 1000)
336
+ log.append(
337
+ f"[{_ts()}] {short}: streaming clip {j + 1}/{n_samples} "
338
+ f"({len(audio_bytes) // 1024}KB, {elapsed}ms so far)"
339
+ )
340
+
341
  return result
342
 
343
  with concurrent.futures.ThreadPoolExecutor(max_workers=1) as ex:
 
345
  try:
346
  arrays = fut.result(timeout=STREAMING_TIMEOUT)
347
  total_ms = int((time.time() - t0) * 1000)
348
+ log.append(
349
+ f"[{_ts()}] {short}: βœ“ streaming done, "
350
+ f"{len(arrays)} clips, total={total_ms}ms"
351
+ )
352
  return repo, arrays
353
  except concurrent.futures.TimeoutError:
354
  total_ms = int((time.time() - t0) * 1000)
355
+ log.append(
356
+ f"[{_ts()}] {short}: βœ— streaming timeout after "
357
+ f"{STREAMING_TIMEOUT}s β€” skipping (total={total_ms}ms)"
358
+ )
359
  return repo, []
360
 
361
  except Exception as exc:
362
+ log.append(f"[{_ts()}] {short}: βœ— {exc} (total={int((time.time() - t0) * 1000)}ms)")
363
  return repo, []
364
 
365
 
 
382
  log: list[str] = []
383
  t_total = time.time()
384
 
385
+ log.append(
386
+ f"[{_ts()}] === START: {len(repos)} repos, "
387
+ f"{samples_per_book} sample/book, {audio_sec}s/clip ==="
388
+ )
389
  log.append(f"[{_ts()}] CPU count: {N_CPUS}, workers: {N_WORKERS}, embed_threads: {EMBED_THREADS}")
390
 
391
  progress(0, desc="Loading model…")
392
  t_model = time.time()
393
  _load_model()
394
+ log.append(f"[{_ts()}] model loaded in {int((time.time() - t_model) * 1000)}ms")
395
 
396
  # ── Phase 1: download all audio in parallel (I/O bound) ──────────────────
397
+ log.append(f"[{_ts()}] --- Phase 1: download ({N_CPUS * 2} workers) ---")
398
  progress(0.05, desc=f"Downloading audio from {len(repos)} datasets…")
399
  repo_arrays: dict[str, list[np.ndarray]] = {}
400
 
 
407
  for future in concurrent.futures.as_completed(futures):
408
  repo, arrays = future.result()
409
  done += 1
410
+ progress(
411
+ 0.05 + 0.55 * done / len(repos),
412
+ desc=f"[{done}/{len(repos)}] {repo.split('/')[-1]}",
413
+ )
414
  if arrays:
415
  repo_arrays[repo] = arrays
416
+ else:
417
+ log.append(f"[{_ts()}] {repo.split('/')[-1]}: no usable audio clips after loading")
418
 
419
  ok = len(repo_arrays)
420
  failed = len(repos) - ok
421
+ log.append(
422
+ f"[{_ts()}] Phase 1 done: {ok} ok, {failed} failed, "
423
+ f"phase_total={int((time.time() - t_total) * 1000)}ms"
424
+ )
425
 
426
  if not repo_arrays:
427
  return pd.DataFrame(), "No audio downloaded.", "\n".join(log), "", None
 
437
  all_arrays.extend(repo_arrays[repo])
438
  repo_slices[repo] = (start, len(all_arrays))
439
 
440
+ try:
441
+ all_embeddings = _parallel_embed(all_arrays, log)
442
+ except Exception as exc:
443
+ log.append(f"[{_ts()}] βœ— embedding failed: {exc}")
444
+ return pd.DataFrame(), f"Embedding failed: {exc}", "\n".join(log), "", None
445
 
446
  # ── Phase 3: average per repo, cluster ───────────────────────────────────
447
  log.append(f"[{_ts()}] --- Phase 3: cluster ({len(repo_slices)} repos) ---")
448
  progress(0.95, desc="Clustering…")
449
+
450
  embeddings: dict[str, np.ndarray] = {}
451
  for repo, (start, end) in repo_slices.items():
452
+ if end > start:
453
+ embeddings[repo] = all_embeddings[start:end].mean(axis=0)
454
+
455
+ if not embeddings:
456
+ return pd.DataFrame(), "No embeddings created.", "\n".join(log), "", None
457
 
458
  repo_names = list(embeddings.keys())
459
  emb_matrix = np.stack([embeddings[r] for r in repo_names])
460
+
461
+ norms = np.linalg.norm(emb_matrix, axis=1, keepdims=True)
462
+ norms = np.where(norms == 0, 1.0, norms)
463
+ emb_matrix = emb_matrix / norms
464
+
465
  sim_matrix = np.clip(emb_matrix @ emb_matrix.T, -1.0, 1.0)
466
  dist_matrix = 1.0 - sim_matrix
467
  np.fill_diagonal(dist_matrix, 0.0)
 
480
  rows = []
481
  for i, repo in enumerate(repo_names):
482
  cluster = labels[i]
483
+ same_idx = [j for j, label in enumerate(labels) if label == cluster and j != i]
484
  intra_sim = float(np.mean([sim_matrix[i][j] for j in same_idx])) if same_idx else 1.0
485
+
486
  other_sorted = sorted([j for j in range(n) if j != i], key=lambda j: -sim_matrix[i][j])
487
  closest = (
488
  f"{repo_names[other_sorted[0]].split('/')[-1]} ({sim_matrix[i][other_sorted[0]]:.2f})"
489
+ if other_sorted
490
+ else "-"
491
+ )
492
+
493
+ rows.append(
494
+ {
495
+ "dataset": repo.split("/")[-1],
496
+ "speaker_id": f"speaker_{cluster + 1:02d}",
497
+ "books_with_speaker": sum(1 for label in labels if label == cluster),
498
+ "intra_sim": round(intra_sim, 3),
499
+ "closest_match": closest,
500
+ }
501
  )
 
 
 
 
 
 
 
502
 
503
  df = pd.DataFrame(rows).sort_values(["speaker_id", "dataset"]).reset_index(drop=True)
504
  n_speakers = len(set(labels))
505
  total_ms = int((time.time() - t_total) * 1000)
506
+ summary = f"βœ… {len(repo_names)} books β†’ {n_speakers} unique speakers ({total_ms / 1000:.1f}s total)"
507
  log.append(f"[{_ts()}] === DONE: {total_ms}ms total ===")
508
 
509
  # Plain-text copy-friendly output (space-separated, matches log format)
510
  text_lines = []
511
  for _, r in df.iterrows():
512
  text_lines.append(
513
+ f"{r['dataset']} {r['speaker_id']} {r['books_with_speaker']} "
514
+ f"{r['intra_sim']} {r['closest_match']}"
515
  )
516
  plain_text = "\n".join(text_lines)
517
 
 
528
 
529
  Finds unique speakers across multiple HF audio datasets. Each dataset is assumed to have
530
  **one speaker** (e.g. an audiobook). The app fetches audio directly via the datasets-server API
531
+ (no full parquet download), downloads in parallel, then embeds clips across parallel workers.
532
 
533
  **Model:** [microsoft/wavlm-base-plus-sv](https://huggingface.co/microsoft/wavlm-base-plus-sv) β€”
534
  language-agnostic speaker embeddings, works for any language.
 
543
  - *Audio length (sec)* β€” seconds of each chunk to use for embedding. 5 sec is sufficient for a clear voice.
544
  - *Same-speaker threshold* β€” cosine similarity cutoff. Raise if too many books merge into one speaker; lower if one person gets split across IDs.
545
  - *HF Token* β€” only needed for **private** repos.
546
+ 3. Click **Identify Speakers**. Downloads run in parallel, then clips are embedded in parallel.
547
 
548
  ## Output columns
549
 
 
560
  The **Errors / Timing** box shows per-dataset timing and batch embed stats β€” useful for diagnosing slow datasets.
561
  """
562
 
563
+
564
  with gr.Blocks(title="Speaker Identifier") as demo:
565
  gr.Markdown(DESCRIPTION)
566
  gr.Markdown("---")
567
+
568
  with gr.Row():
569
  with gr.Column(scale=2):
570
  repo_input = gr.Textbox(
 
575
  with gr.Column(scale=1):
576
  samples = gr.Slider(1, 10, value=3, step=1, label="Samples per book")
577
  audio_sec = gr.Slider(
578
+ 2,
579
+ 30,
580
+ value=5,
581
+ step=1,
582
  label="Audio length per sample (sec)",
583
  info="5 sec is usually enough; longer = more accurate but slower",
584
  )
585
  threshold = gr.Slider(
586
+ 0.60,
587
+ 0.98,
588
+ value=0.82,
589
+ step=0.01,
590
  label="Same-speaker threshold",
591
  info="Higher = stricter matching β†’ more clusters",
592
  )
 
603
  headers=["dataset", "speaker_id", "books_with_speaker", "intra_sim", "closest_match"],
604
  wrap=True,
605
  )
606
+
607
  with gr.Row():
608
  text_out = gr.Textbox(
609
  label="Plain text (copy-friendly)",
 
612
  info="dataset speaker_id n_books intra_sim closest_match",
613
  )
614
  csv_out = gr.File(label="Download CSV", file_types=[".csv"])
615
+
616
  errors_out = gr.Textbox(label="Errors / Timing", interactive=False)
617
 
618
  run_btn.click(
 
621
  outputs=[table_out, summary_out, errors_out, text_out, csv_out],
622
  )
623
 
624
+
625
  demo.launch()