Claude commited on
Commit
4a08059
·
1 Parent(s): dd5c0bd

Add audio transcription + speaker diarization tab (Whisper + pyannote)

Browse files
Files changed (2) hide show
  1. app.py +109 -0
  2. requirements.txt +1 -0
app.py CHANGED
@@ -31,6 +31,8 @@ _adapter: AnimateDiffAdapter | None = None
31
  _sd_pipe = None
32
  _ltx_pipe = None
33
  _wan_pixel_pipe = None
 
 
34
 
35
 
36
  def _get_adapter() -> AnimateDiffAdapter:
@@ -123,6 +125,87 @@ def _get_wan_pixel_pipe():
123
  return pipe
124
 
125
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
126
  @spaces.GPU(duration=90)
127
  def infer(
128
  image: np.ndarray,
@@ -451,6 +534,32 @@ with gr.Blocks(title="Venture-Studio") as demo:
451
  api_name="infer_wan_pixel",
452
  )
453
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
454
 
455
  if __name__ == "__main__":
456
  demo.launch()
 
31
  _sd_pipe = None
32
  _ltx_pipe = None
33
  _wan_pixel_pipe = None
34
+ _asr_pipe = None
35
+ _diar_pipe = None
36
 
37
 
38
  def _get_adapter() -> AnimateDiffAdapter:
 
125
  return pipe
126
 
127
 
128
+ def _get_asr_pipe():
129
+ """Lazily load a GPU Whisper ASR pipeline (transformers, already a dependency)."""
130
+ global _asr_pipe
131
+ if _asr_pipe is not None:
132
+ return _asr_pipe
133
+ import torch
134
+ from transformers import pipeline as hf_pipeline
135
+
136
+ _asr_pipe = hf_pipeline(
137
+ "automatic-speech-recognition",
138
+ model="openai/whisper-small",
139
+ torch_dtype=torch.float16,
140
+ device="cuda",
141
+ return_timestamps=True,
142
+ )
143
+ print("Whisper (small) ASR pipeline loaded on cuda", flush=True)
144
+ return _asr_pipe
145
+
146
+
147
+ def _get_diar_pipe():
148
+ """Lazily load the pyannote speaker-diarization pipeline.
149
+
150
+ Requires the `HF_TOKEN` Space secret (set via the Spaces UI/API, never
151
+ committed to this repo) — the token must have accepted the gated terms
152
+ for pyannote/speaker-diarization-3.1 and pyannote/segmentation-3.0, and
153
+ have "read access to public gated repos" enabled in its token settings.
154
+ """
155
+ global _diar_pipe
156
+ if _diar_pipe is not None:
157
+ return _diar_pipe
158
+ import os
159
+ from pyannote.audio import Pipeline as DiarPipeline
160
+
161
+ token = os.environ.get("HF_TOKEN")
162
+ _diar_pipe = DiarPipeline.from_pretrained(
163
+ "pyannote/speaker-diarization-3.1", use_auth_token=token
164
+ )
165
+ _diar_pipe.to_device = getattr(_diar_pipe, "to_device", None)
166
+ try:
167
+ import torch
168
+ _diar_pipe.to(torch.device("cuda"))
169
+ except Exception as e:
170
+ print(f"WARN: could not move diarization pipeline to cuda: {e}", flush=True)
171
+ print("pyannote speaker-diarization-3.1 pipeline loaded", flush=True)
172
+ return _diar_pipe
173
+
174
+
175
+ @spaces.GPU(duration=300)
176
+ def infer_audio(audio_path: str) -> tuple[str, str]:
177
+ """Transcribe + diarize an uploaded audio/video file.
178
+
179
+ Returns (transcript_text, diarization_json_text). Transcript segments are
180
+ timestamped; diarization turns are (start, end, speaker_label) — labels
181
+ are anonymous (SPEAKER_00, SPEAKER_01, ...), not real identities.
182
+ """
183
+ import json
184
+
185
+ asr = _get_asr_pipe()
186
+ asr_result = asr(audio_path, chunk_length_s=30, batch_size=8)
187
+
188
+ lines = []
189
+ for chunk in asr_result.get("chunks", []):
190
+ start, end = chunk.get("timestamp", (None, None))
191
+ text = chunk.get("text", "").strip()
192
+ if start is None:
193
+ lines.append(text)
194
+ else:
195
+ lines.append(f"[{start:07.2f} -> {end:07.2f}] {text}")
196
+ transcript_text = "\n".join(lines) if lines else asr_result.get("text", "")
197
+
198
+ diar = _get_diar_pipe()
199
+ diarization = diar(audio_path)
200
+ turns = [
201
+ {"start": round(turn.start, 2), "end": round(turn.end, 2), "speaker": speaker}
202
+ for turn, _, speaker in diarization.itertracks(yield_label=True)
203
+ ]
204
+ diarization_json = json.dumps(turns, indent=2)
205
+
206
+ return transcript_text, diarization_json
207
+
208
+
209
  @spaces.GPU(duration=90)
210
  def infer(
211
  image: np.ndarray,
 
534
  api_name="infer_wan_pixel",
535
  )
536
 
537
+ with gr.Tab("Audio (Whisper + Diarization)"):
538
+ gr.Markdown(
539
+ "### Whisper transcription + pyannote speaker diarization\n"
540
+ "Upload an audio or video file. Returns a timestamped transcript "
541
+ "and anonymous speaker turns (SPEAKER_00, SPEAKER_01, ...) — this "
542
+ "does NOT identify real names, only separates who-spoke-when."
543
+ )
544
+ with gr.Row():
545
+ with gr.Column():
546
+ audio_in = gr.Audio(label="Source audio/video", type="filepath")
547
+ audio_run = gr.Button("Transcribe + diarize", variant="primary")
548
+ with gr.Column():
549
+ transcript_out = gr.Textbox(
550
+ label="Timestamped transcript", lines=20, show_copy_button=True
551
+ )
552
+ diarization_out = gr.Textbox(
553
+ label="Speaker turns (JSON)", lines=20, show_copy_button=True
554
+ )
555
+
556
+ audio_run.click(
557
+ infer_audio,
558
+ inputs=[audio_in],
559
+ outputs=[transcript_out, diarization_out],
560
+ api_name="infer_audio",
561
+ )
562
+
563
 
564
  if __name__ == "__main__":
565
  demo.launch()
requirements.txt CHANGED
@@ -14,3 +14,4 @@ accelerate>=0.30
14
  peft>=0.10
15
  spaces>=0.30
16
  sentencepiece>=0.2
 
 
14
  peft>=0.10
15
  spaces>=0.30
16
  sentencepiece>=0.2
17
+ pyannote.audio>=3.1