Claude commited on
Commit ·
4a08059
1
Parent(s): dd5c0bd
Add audio transcription + speaker diarization tab (Whisper + pyannote)
Browse files- app.py +109 -0
- requirements.txt +1 -0
app.py
CHANGED
|
@@ -31,6 +31,8 @@ _adapter: AnimateDiffAdapter | None = None
|
|
| 31 |
_sd_pipe = None
|
| 32 |
_ltx_pipe = None
|
| 33 |
_wan_pixel_pipe = None
|
|
|
|
|
|
|
| 34 |
|
| 35 |
|
| 36 |
def _get_adapter() -> AnimateDiffAdapter:
|
|
@@ -123,6 +125,87 @@ def _get_wan_pixel_pipe():
|
|
| 123 |
return pipe
|
| 124 |
|
| 125 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
@spaces.GPU(duration=90)
|
| 127 |
def infer(
|
| 128 |
image: np.ndarray,
|
|
@@ -451,6 +534,32 @@ with gr.Blocks(title="Venture-Studio") as demo:
|
|
| 451 |
api_name="infer_wan_pixel",
|
| 452 |
)
|
| 453 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 454 |
|
| 455 |
if __name__ == "__main__":
|
| 456 |
demo.launch()
|
|
|
|
| 31 |
_sd_pipe = None
|
| 32 |
_ltx_pipe = None
|
| 33 |
_wan_pixel_pipe = None
|
| 34 |
+
_asr_pipe = None
|
| 35 |
+
_diar_pipe = None
|
| 36 |
|
| 37 |
|
| 38 |
def _get_adapter() -> AnimateDiffAdapter:
|
|
|
|
| 125 |
return pipe
|
| 126 |
|
| 127 |
|
| 128 |
+
def _get_asr_pipe():
|
| 129 |
+
"""Lazily load a GPU Whisper ASR pipeline (transformers, already a dependency)."""
|
| 130 |
+
global _asr_pipe
|
| 131 |
+
if _asr_pipe is not None:
|
| 132 |
+
return _asr_pipe
|
| 133 |
+
import torch
|
| 134 |
+
from transformers import pipeline as hf_pipeline
|
| 135 |
+
|
| 136 |
+
_asr_pipe = hf_pipeline(
|
| 137 |
+
"automatic-speech-recognition",
|
| 138 |
+
model="openai/whisper-small",
|
| 139 |
+
torch_dtype=torch.float16,
|
| 140 |
+
device="cuda",
|
| 141 |
+
return_timestamps=True,
|
| 142 |
+
)
|
| 143 |
+
print("Whisper (small) ASR pipeline loaded on cuda", flush=True)
|
| 144 |
+
return _asr_pipe
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _get_diar_pipe():
|
| 148 |
+
"""Lazily load the pyannote speaker-diarization pipeline.
|
| 149 |
+
|
| 150 |
+
Requires the `HF_TOKEN` Space secret (set via the Spaces UI/API, never
|
| 151 |
+
committed to this repo) — the token must have accepted the gated terms
|
| 152 |
+
for pyannote/speaker-diarization-3.1 and pyannote/segmentation-3.0, and
|
| 153 |
+
have "read access to public gated repos" enabled in its token settings.
|
| 154 |
+
"""
|
| 155 |
+
global _diar_pipe
|
| 156 |
+
if _diar_pipe is not None:
|
| 157 |
+
return _diar_pipe
|
| 158 |
+
import os
|
| 159 |
+
from pyannote.audio import Pipeline as DiarPipeline
|
| 160 |
+
|
| 161 |
+
token = os.environ.get("HF_TOKEN")
|
| 162 |
+
_diar_pipe = DiarPipeline.from_pretrained(
|
| 163 |
+
"pyannote/speaker-diarization-3.1", use_auth_token=token
|
| 164 |
+
)
|
| 165 |
+
_diar_pipe.to_device = getattr(_diar_pipe, "to_device", None)
|
| 166 |
+
try:
|
| 167 |
+
import torch
|
| 168 |
+
_diar_pipe.to(torch.device("cuda"))
|
| 169 |
+
except Exception as e:
|
| 170 |
+
print(f"WARN: could not move diarization pipeline to cuda: {e}", flush=True)
|
| 171 |
+
print("pyannote speaker-diarization-3.1 pipeline loaded", flush=True)
|
| 172 |
+
return _diar_pipe
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
@spaces.GPU(duration=300)
|
| 176 |
+
def infer_audio(audio_path: str) -> tuple[str, str]:
|
| 177 |
+
"""Transcribe + diarize an uploaded audio/video file.
|
| 178 |
+
|
| 179 |
+
Returns (transcript_text, diarization_json_text). Transcript segments are
|
| 180 |
+
timestamped; diarization turns are (start, end, speaker_label) — labels
|
| 181 |
+
are anonymous (SPEAKER_00, SPEAKER_01, ...), not real identities.
|
| 182 |
+
"""
|
| 183 |
+
import json
|
| 184 |
+
|
| 185 |
+
asr = _get_asr_pipe()
|
| 186 |
+
asr_result = asr(audio_path, chunk_length_s=30, batch_size=8)
|
| 187 |
+
|
| 188 |
+
lines = []
|
| 189 |
+
for chunk in asr_result.get("chunks", []):
|
| 190 |
+
start, end = chunk.get("timestamp", (None, None))
|
| 191 |
+
text = chunk.get("text", "").strip()
|
| 192 |
+
if start is None:
|
| 193 |
+
lines.append(text)
|
| 194 |
+
else:
|
| 195 |
+
lines.append(f"[{start:07.2f} -> {end:07.2f}] {text}")
|
| 196 |
+
transcript_text = "\n".join(lines) if lines else asr_result.get("text", "")
|
| 197 |
+
|
| 198 |
+
diar = _get_diar_pipe()
|
| 199 |
+
diarization = diar(audio_path)
|
| 200 |
+
turns = [
|
| 201 |
+
{"start": round(turn.start, 2), "end": round(turn.end, 2), "speaker": speaker}
|
| 202 |
+
for turn, _, speaker in diarization.itertracks(yield_label=True)
|
| 203 |
+
]
|
| 204 |
+
diarization_json = json.dumps(turns, indent=2)
|
| 205 |
+
|
| 206 |
+
return transcript_text, diarization_json
|
| 207 |
+
|
| 208 |
+
|
| 209 |
@spaces.GPU(duration=90)
|
| 210 |
def infer(
|
| 211 |
image: np.ndarray,
|
|
|
|
| 534 |
api_name="infer_wan_pixel",
|
| 535 |
)
|
| 536 |
|
| 537 |
+
with gr.Tab("Audio (Whisper + Diarization)"):
|
| 538 |
+
gr.Markdown(
|
| 539 |
+
"### Whisper transcription + pyannote speaker diarization\n"
|
| 540 |
+
"Upload an audio or video file. Returns a timestamped transcript "
|
| 541 |
+
"and anonymous speaker turns (SPEAKER_00, SPEAKER_01, ...) — this "
|
| 542 |
+
"does NOT identify real names, only separates who-spoke-when."
|
| 543 |
+
)
|
| 544 |
+
with gr.Row():
|
| 545 |
+
with gr.Column():
|
| 546 |
+
audio_in = gr.Audio(label="Source audio/video", type="filepath")
|
| 547 |
+
audio_run = gr.Button("Transcribe + diarize", variant="primary")
|
| 548 |
+
with gr.Column():
|
| 549 |
+
transcript_out = gr.Textbox(
|
| 550 |
+
label="Timestamped transcript", lines=20, show_copy_button=True
|
| 551 |
+
)
|
| 552 |
+
diarization_out = gr.Textbox(
|
| 553 |
+
label="Speaker turns (JSON)", lines=20, show_copy_button=True
|
| 554 |
+
)
|
| 555 |
+
|
| 556 |
+
audio_run.click(
|
| 557 |
+
infer_audio,
|
| 558 |
+
inputs=[audio_in],
|
| 559 |
+
outputs=[transcript_out, diarization_out],
|
| 560 |
+
api_name="infer_audio",
|
| 561 |
+
)
|
| 562 |
+
|
| 563 |
|
| 564 |
if __name__ == "__main__":
|
| 565 |
demo.launch()
|
requirements.txt
CHANGED
|
@@ -14,3 +14,4 @@ accelerate>=0.30
|
|
| 14 |
peft>=0.10
|
| 15 |
spaces>=0.30
|
| 16 |
sentencepiece>=0.2
|
|
|
|
|
|
| 14 |
peft>=0.10
|
| 15 |
spaces>=0.30
|
| 16 |
sentencepiece>=0.2
|
| 17 |
+
pyannote.audio>=3.1
|