Upload folder using huggingface_hub

Dockerfile

@@ -1,17 +1,17 @@
-FROM pytorch/pytorch:2.4.0-cuda12.4-cudnn9-runtime
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    ffmpeg libsndfile1 git \
-    && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /app
-
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
-
-COPY app.py .
-
-# HF Inference Endpoints require port 80
-EXPOSE 80
-
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "80"]
+FROM pytorch/pytorch:2.4.0-cuda12.4-cudnn9-runtime
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ffmpeg libsndfile1 git \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY app.py .
+
+# HF Inference Endpoints require port 80
+EXPOSE 80
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "80"]

README.md

@@ -1,84 +1,84 @@
----
-tags:
-  - audio
-  - speaker-diarization
-  - speaker-embedding
-  - pyannote
-  - funasr
-  - meetingmind
-library_name: custom
-pipeline_tag: audio-classification
----
-
-# MeetingMind GPU Service
-
-GPU-accelerated speaker diarization and embedding extraction for the MeetingMind pipeline. Runs as an HF Inference Endpoint on a T4 GPU with scale-to-zero.
-
-## API
-
-### `GET /health`
-
-Returns service status and GPU availability.
-
-```bash
-curl -H "Authorization: Bearer $HF_TOKEN" $ENDPOINT_URL/health
-```
-
-```json
-{"status": "ok", "gpu_available": true}
-```
-
-### `POST /diarize`
-
-Speaker diarization using pyannote v4. Accepts any audio format (FLAC, WAV, MP3, etc.).
-
-```bash
-curl -X POST \
-  -H "Authorization: Bearer $HF_TOKEN" \
-  -F audio=@meeting.flac \
-  -F min_speakers=2 \
-  -F max_speakers=6 \
-  $ENDPOINT_URL/diarize
-```
-
-```json
-{
-  "segments": [
-    {"speaker": "SPEAKER_00", "start": 0.5, "end": 3.2, "duration": 2.7},
-    {"speaker": "SPEAKER_01", "start": 3.4, "end": 7.1, "duration": 3.7}
-  ]
-}
-```
-
-### `POST /embed`
-
-Speaker embedding extraction using FunASR CAM++. Returns L2-normalized 192-dim vectors for voiceprint matching.
-
-```bash
-curl -X POST \
-  -H "Authorization: Bearer $HF_TOKEN" \
-  -F audio=@meeting.flac \
-  -F start_time=1.0 \
-  -F end_time=5.0 \
-  $ENDPOINT_URL/embed
-```
-
-```json
-{"embedding": [0.012, -0.034, ...], "dim": 192}
-```
-
-## Environment Variables
-
-| Variable | Default | Description |
-|---|---|---|
-| `HF_TOKEN` | (required) | Hugging Face token for pyannote model access |
-| `PYANNOTE_MIN_SPEAKERS` | `1` | Minimum speakers for diarization |
-| `PYANNOTE_MAX_SPEAKERS` | `10` | Maximum speakers for diarization |
-
-## Architecture
-
-- **Base image**: `pytorch/pytorch:2.4.0-cuda12.4-cudnn9-runtime`
-- **Diarization**: pyannote/speaker-diarization-community-1 (~2GB VRAM)
-- **Embeddings**: FunASR CAM++ sv_zh-cn_16k-common (~200MB)
-- **Total VRAM**: ~3GB (fits T4 16GB with headroom)
-- **Scale-to-zero**: 15 min idle timeout (~$0.60/hr when active)
+---
+tags:
+  - audio
+  - speaker-diarization
+  - speaker-embedding
+  - pyannote
+  - funasr
+  - meetingmind
+library_name: custom
+pipeline_tag: audio-classification
+---
+
+# MeetingMind GPU Service
+
+GPU-accelerated speaker diarization and embedding extraction for the MeetingMind pipeline. Runs as an HF Inference Endpoint on a T4 GPU with scale-to-zero.
+
+## API
+
+### `GET /health`
+
+Returns service status and GPU availability.
+
+```bash
+curl -H "Authorization: Bearer $HF_TOKEN" $ENDPOINT_URL/health
+```
+
+```json
+{"status": "ok", "gpu_available": true}
+```
+
+### `POST /diarize`
+
+Speaker diarization using pyannote v4. Accepts any audio format (FLAC, WAV, MP3, etc.).
+
+```bash
+curl -X POST \
+  -H "Authorization: Bearer $HF_TOKEN" \
+  -F audio=@meeting.flac \
+  -F min_speakers=2 \
+  -F max_speakers=6 \
+  $ENDPOINT_URL/diarize
+```
+
+```json
+{
+  "segments": [
+    {"speaker": "SPEAKER_00", "start": 0.5, "end": 3.2, "duration": 2.7},
+    {"speaker": "SPEAKER_01", "start": 3.4, "end": 7.1, "duration": 3.7}
+  ]
+}
+```
+
+### `POST /embed`
+
+Speaker embedding extraction using FunASR CAM++. Returns L2-normalized 192-dim vectors for voiceprint matching.
+
+```bash
+curl -X POST \
+  -H "Authorization: Bearer $HF_TOKEN" \
+  -F audio=@meeting.flac \
+  -F start_time=1.0 \
+  -F end_time=5.0 \
+  $ENDPOINT_URL/embed
+```
+
+```json
+{"embedding": [0.012, -0.034, ...], "dim": 192}
+```
+
+## Environment Variables
+
+| Variable | Default | Description |
+|---|---|---|
+| `HF_TOKEN` | (required) | Hugging Face token for pyannote model access |
+| `PYANNOTE_MIN_SPEAKERS` | `1` | Minimum speakers for diarization |
+| `PYANNOTE_MAX_SPEAKERS` | `10` | Maximum speakers for diarization |
+
+## Architecture
+
+- **Base image**: `pytorch/pytorch:2.4.0-cuda12.4-cudnn9-runtime`
+- **Diarization**: pyannote/speaker-diarization-community-1 (~2GB VRAM)
+- **Embeddings**: FunASR CAM++ sv_zh-cn_16k-common (~200MB)
+- **Total VRAM**: ~3GB (fits T4 16GB with headroom)
+- **Scale-to-zero**: 15 min idle timeout (~$0.60/hr when active)

app.py

@@ -1,273 +1,273 @@
-"""
-Slim GPU service for HF Inference Endpoints.
-Exposes /diarize, /embed, /transcribe, and /transcribe/stream endpoints.
-"""
-
-import io
-import json
-import logging
-import os
-import re
-import threading
-from contextlib import asynccontextmanager
-
-import numpy as np
-import soundfile as sf
-import librosa
-import torch
-from fastapi import FastAPI, File, Form, UploadFile
-from fastapi.responses import JSONResponse
-from pydub import AudioSegment
-from sse_starlette.sse import EventSourceResponse
-
-logger = logging.getLogger("gpu_service")
-
-# ---------------------------------------------------------------------------
-# Config
-# ---------------------------------------------------------------------------
-HF_TOKEN = os.environ.get("HF_TOKEN", "")
-PYANNOTE_MODEL = "pyannote/speaker-diarization-community-1"
-FUNASR_MODEL = "iic/speech_campplus_sv_zh-cn_16k-common"
-PYANNOTE_MIN_SPEAKERS = int(os.environ.get("PYANNOTE_MIN_SPEAKERS", "1"))
-PYANNOTE_MAX_SPEAKERS = int(os.environ.get("PYANNOTE_MAX_SPEAKERS", "10"))
-TARGET_SR = 16000
-
-# ---------------------------------------------------------------------------
-# Singletons
-# ---------------------------------------------------------------------------
-_diarize_pipeline = None
-_embed_model = None
-_voxtral_model = None
-_voxtral_processor = None
-
-VOXTRAL_MODEL_ID = "mistralai/Voxtral-Mini-4B-Realtime-2602"
-
-# Markers to strip from Voxtral output
-_MARKER_RE = re.compile(r"\[STREAMING_PAD\]|\[STREAMING_WORD\]")
-
-
-def _load_diarize_pipeline():
-    global _diarize_pipeline
-    if _diarize_pipeline is None:
-        from pyannote.audio import Pipeline as PyannotePipeline
-
-        _diarize_pipeline = PyannotePipeline.from_pretrained(
-            PYANNOTE_MODEL, token=HF_TOKEN
-        )
-        _diarize_pipeline = _diarize_pipeline.to(torch.device("cuda"))
-    return _diarize_pipeline
-
-
-def _load_embed_model():
-    global _embed_model
-    if _embed_model is None:
-        from funasr import AutoModel
-
-        _embed_model = AutoModel(model=FUNASR_MODEL)
-    return _embed_model
-
-
-def _load_voxtral():
-    """Lazy-load Voxtral model and processor (first call only)."""
-    global _voxtral_model, _voxtral_processor
-    if _voxtral_model is None:
-        from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
-
-        logger.info("Loading Voxtral model %s ...", VOXTRAL_MODEL_ID)
-        _voxtral_processor = AutoProcessor.from_pretrained(
-            VOXTRAL_MODEL_ID, trust_remote_code=True
-        )
-        _voxtral_model = AutoModelForSpeechSeq2Seq.from_pretrained(
-            VOXTRAL_MODEL_ID, torch_dtype=torch.float16, trust_remote_code=True
-        ).to("cuda")
-        logger.info("Voxtral model loaded.")
-    return _voxtral_model, _voxtral_processor
-
-
-def _clean_voxtral_text(text: str) -> str:
-    """Strip Voxtral streaming markers and collapse whitespace."""
-    text = _MARKER_RE.sub("", text)
-    return " ".join(text.split()).strip()
-
-
-# ---------------------------------------------------------------------------
-# Audio helpers
-# ---------------------------------------------------------------------------
-def prepare_audio(raw_bytes: bytes) -> np.ndarray:
-    """Read any audio format -> float32 mono @ 16 kHz."""
-    audio, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")
-    if audio.ndim > 1:
-        audio = audio.mean(axis=1)
-    if sr != TARGET_SR:
-        audio = librosa.resample(audio, orig_sr=sr, target_sr=TARGET_SR)
-    return audio
-
-
-def prepare_audio_slice(raw_bytes: bytes, start_time: float, end_time: float) -> np.ndarray:
-    """Read audio, slice by time, return float32 mono @ 16 kHz."""
-    seg = AudioSegment.from_file(io.BytesIO(raw_bytes))
-    seg = seg[int(start_time * 1000):int(end_time * 1000)]
-    seg = seg.set_frame_rate(TARGET_SR).set_channels(1).set_sample_width(2)
-    return np.array(seg.get_array_of_samples(), dtype=np.float32) / 32768.0
-
-
-# ---------------------------------------------------------------------------
-# App
-# ---------------------------------------------------------------------------
-@asynccontextmanager
-async def lifespan(app: FastAPI):
-    # Warm up diarization pipeline at startup (embedding model lazy-loads)
-    _load_diarize_pipeline()
-    yield
-
-
-app = FastAPI(title="GPU Service (HF Endpoint)", lifespan=lifespan)
-
-
-@app.get("/health")
-async def health():
-    return {"status": "ok", "gpu_available": torch.cuda.is_available()}
-
-
-@app.post("/diarize")
-async def diarize(
-    audio: UploadFile = File(...),
-    min_speakers: int | None = Form(None),
-    max_speakers: int | None = Form(None),
-):
-    try:
-        raw = await audio.read()
-        audio_16k = prepare_audio(raw)
-
-        pipeline = _load_diarize_pipeline()
-        waveform = torch.from_numpy(audio_16k).unsqueeze(0).float()
-        input_data = {"waveform": waveform, "sample_rate": TARGET_SR}
-
-        result = pipeline(
-            input_data,
-            min_speakers=min_speakers or PYANNOTE_MIN_SPEAKERS,
-            max_speakers=max_speakers or PYANNOTE_MAX_SPEAKERS,
-        )
-        # pyannote v4 compat
-        diarization = getattr(result, "speaker_diarization", result)
-
-        segments = []
-        for turn, _, speaker in diarization.itertracks(yield_label=True):
-            segments.append(
-                {
-                    "speaker": speaker,
-                    "start": round(turn.start, 3),
-                    "end": round(turn.end, 3),
-                    "duration": round(turn.end - turn.start, 3),
-                }
-            )
-        segments.sort(key=lambda s: s["start"])
-        return {"segments": segments}
-    except Exception as e:
-        return JSONResponse(status_code=500, content={"error": str(e)})
-
-
-@app.post("/embed")
-async def embed(
-    audio: UploadFile = File(...),
-    start_time: float | None = Form(None),
-    end_time: float | None = Form(None),
-):
-    try:
-        raw = await audio.read()
-        if start_time is not None and end_time is not None:
-            audio_16k = prepare_audio_slice(raw, start_time, end_time)
-        else:
-            audio_16k = prepare_audio(raw)
-
-        model = _load_embed_model()
-        result = model.generate(input=audio_16k, output_dir=None)
-        raw_emb = result[0]["spk_embedding"]
-        if hasattr(raw_emb, "cpu"):
-            raw_emb = raw_emb.cpu().numpy()
-        emb = np.array(raw_emb).flatten()
-
-        # L2-normalize
-        norm = np.linalg.norm(emb)
-        if norm > 0:
-            emb = emb / norm
-
-        return {"embedding": emb.tolist(), "dim": len(emb)}
-    except Exception as e:
-        return JSONResponse(status_code=500, content={"error": str(e)})
-
-
-@app.post("/transcribe")
-async def transcribe(
-    audio: UploadFile = File(...),
-    prompt: str = Form("Transcribe this audio."),
-):
-    try:
-        raw = await audio.read()
-        audio_16k = prepare_audio(raw)
-
-        model, processor = _load_voxtral()
-        inputs = processor(
-            audios=audio_16k,
-            sampling_rate=TARGET_SR,
-            text=prompt,
-            return_tensors="pt",
-        ).to("cuda")
-
-        output_ids = model.generate(**inputs, max_new_tokens=1024)
-        text = processor.batch_decode(output_ids, skip_special_tokens=True)[0]
-        text = _clean_voxtral_text(text)
-
-        return {"text": text}
-    except Exception as e:
-        logger.exception("Transcription failed")
-        return JSONResponse(status_code=500, content={"error": str(e)})
-
-
-@app.post("/transcribe/stream")
-async def transcribe_stream(
-    audio: UploadFile = File(...),
-    prompt: str = Form("Transcribe this audio."),
-):
-    try:
-        raw = await audio.read()
-        audio_16k = prepare_audio(raw)
-    except Exception as e:
-        logger.exception("Audio preparation failed")
-        return JSONResponse(status_code=500, content={"error": str(e)})
-
-    async def event_generator():
-        try:
-            from transformers import TextIteratorStreamer
-
-            model, processor = _load_voxtral()
-            inputs = processor(
-                audios=audio_16k,
-                sampling_rate=TARGET_SR,
-                text=prompt,
-                return_tensors="pt",
-            ).to("cuda")
-
-            streamer = TextIteratorStreamer(
-                processor.tokenizer, skip_prompt=True, skip_special_tokens=True
-            )
-            gen_kwargs = {**inputs, "max_new_tokens": 1024, "streamer": streamer}
-
-            thread = threading.Thread(target=model.generate, kwargs=gen_kwargs)
-            thread.start()
-
-            full_text = ""
-            for chunk in streamer:
-                chunk = _MARKER_RE.sub("", chunk)
-                if chunk:
-                    full_text += chunk
-                    yield {"event": "token", "data": json.dumps({"token": chunk})}
-
-            thread.join()
-            full_text = " ".join(full_text.split()).strip()
-            yield {"event": "done", "data": json.dumps({"text": full_text})}
-        except Exception as e:
-            logger.exception("Streaming transcription failed")
-            yield {"event": "error", "data": json.dumps({"error": str(e)})}
-
-    return EventSourceResponse(event_generator())
+"""
+Slim GPU service for HF Inference Endpoints.
+Exposes /diarize, /embed, /transcribe, and /transcribe/stream endpoints.
+"""
+
+import io
+import json
+import logging
+import os
+import re
+import threading
+from contextlib import asynccontextmanager
+
+import numpy as np
+import soundfile as sf
+import librosa
+import torch
+from fastapi import FastAPI, File, Form, UploadFile
+from fastapi.responses import JSONResponse
+from pydub import AudioSegment
+from sse_starlette.sse import EventSourceResponse
+
+logger = logging.getLogger("gpu_service")
+
+# ---------------------------------------------------------------------------
+# Config
+# ---------------------------------------------------------------------------
+HF_TOKEN = os.environ.get("HF_TOKEN", "")
+PYANNOTE_MODEL = "pyannote/speaker-diarization-community-1"
+FUNASR_MODEL = "iic/speech_campplus_sv_zh-cn_16k-common"
+PYANNOTE_MIN_SPEAKERS = int(os.environ.get("PYANNOTE_MIN_SPEAKERS", "1"))
+PYANNOTE_MAX_SPEAKERS = int(os.environ.get("PYANNOTE_MAX_SPEAKERS", "10"))
+TARGET_SR = 16000
+
+# ---------------------------------------------------------------------------
+# Singletons
+# ---------------------------------------------------------------------------
+_diarize_pipeline = None
+_embed_model = None
+_voxtral_model = None
+_voxtral_processor = None
+
+VOXTRAL_MODEL_ID = "mistralai/Voxtral-Mini-4B-Realtime-2602"
+
+# Markers to strip from Voxtral output
+_MARKER_RE = re.compile(r"\[STREAMING_PAD\]|\[STREAMING_WORD\]")
+
+
+def _load_diarize_pipeline():
+    global _diarize_pipeline
+    if _diarize_pipeline is None:
+        from pyannote.audio import Pipeline as PyannotePipeline
+
+        _diarize_pipeline = PyannotePipeline.from_pretrained(
+            PYANNOTE_MODEL, token=HF_TOKEN
+        )
+        _diarize_pipeline = _diarize_pipeline.to(torch.device("cuda"))
+    return _diarize_pipeline
+
+
+def _load_embed_model():
+    global _embed_model
+    if _embed_model is None:
+        from funasr import AutoModel
+
+        _embed_model = AutoModel(model=FUNASR_MODEL)
+    return _embed_model
+
+
+def _load_voxtral():
+    """Lazy-load Voxtral model and processor (first call only)."""
+    global _voxtral_model, _voxtral_processor
+    if _voxtral_model is None:
+        from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
+
+        logger.info("Loading Voxtral model %s ...", VOXTRAL_MODEL_ID)
+        _voxtral_processor = AutoProcessor.from_pretrained(
+            VOXTRAL_MODEL_ID, trust_remote_code=True
+        )
+        _voxtral_model = AutoModelForSpeechSeq2Seq.from_pretrained(
+            VOXTRAL_MODEL_ID, torch_dtype=torch.float16, trust_remote_code=True
+        ).to("cuda")
+        logger.info("Voxtral model loaded.")
+    return _voxtral_model, _voxtral_processor
+
+
+def _clean_voxtral_text(text: str) -> str:
+    """Strip Voxtral streaming markers and collapse whitespace."""
+    text = _MARKER_RE.sub("", text)
+    return " ".join(text.split()).strip()
+
+
+# ---------------------------------------------------------------------------
+# Audio helpers
+# ---------------------------------------------------------------------------
+def prepare_audio(raw_bytes: bytes) -> np.ndarray:
+    """Read any audio format -> float32 mono @ 16 kHz."""
+    audio, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")
+    if audio.ndim > 1:
+        audio = audio.mean(axis=1)
+    if sr != TARGET_SR:
+        audio = librosa.resample(audio, orig_sr=sr, target_sr=TARGET_SR)
+    return audio
+
+
+def prepare_audio_slice(raw_bytes: bytes, start_time: float, end_time: float) -> np.ndarray:
+    """Read audio, slice by time, return float32 mono @ 16 kHz."""
+    seg = AudioSegment.from_file(io.BytesIO(raw_bytes))
+    seg = seg[int(start_time * 1000):int(end_time * 1000)]
+    seg = seg.set_frame_rate(TARGET_SR).set_channels(1).set_sample_width(2)
+    return np.array(seg.get_array_of_samples(), dtype=np.float32) / 32768.0
+
+
+# ---------------------------------------------------------------------------
+# App
+# ---------------------------------------------------------------------------
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Warm up diarization pipeline at startup (embedding model lazy-loads)
+    _load_diarize_pipeline()
+    yield
+
+
+app = FastAPI(title="GPU Service (HF Endpoint)", lifespan=lifespan)
+
+
+@app.get("/health")
+async def health():
+    return {"status": "ok", "gpu_available": torch.cuda.is_available()}
+
+
+@app.post("/diarize")
+async def diarize(
+    audio: UploadFile = File(...),
+    min_speakers: int | None = Form(None),
+    max_speakers: int | None = Form(None),
+):
+    try:
+        raw = await audio.read()
+        audio_16k = prepare_audio(raw)
+
+        pipeline = _load_diarize_pipeline()
+        waveform = torch.from_numpy(audio_16k).unsqueeze(0).float()
+        input_data = {"waveform": waveform, "sample_rate": TARGET_SR}
+
+        result = pipeline(
+            input_data,
+            min_speakers=min_speakers or PYANNOTE_MIN_SPEAKERS,
+            max_speakers=max_speakers or PYANNOTE_MAX_SPEAKERS,
+        )
+        # pyannote v4 compat
+        diarization = getattr(result, "speaker_diarization", result)
+
+        segments = []
+        for turn, _, speaker in diarization.itertracks(yield_label=True):
+            segments.append(
+                {
+                    "speaker": speaker,
+                    "start": round(turn.start, 3),
+                    "end": round(turn.end, 3),
+                    "duration": round(turn.end - turn.start, 3),
+                }
+            )
+        segments.sort(key=lambda s: s["start"])
+        return {"segments": segments}
+    except Exception as e:
+        return JSONResponse(status_code=500, content={"error": str(e)})
+
+
+@app.post("/embed")
+async def embed(
+    audio: UploadFile = File(...),
+    start_time: float | None = Form(None),
+    end_time: float | None = Form(None),
+):
+    try:
+        raw = await audio.read()
+        if start_time is not None and end_time is not None:
+            audio_16k = prepare_audio_slice(raw, start_time, end_time)
+        else:
+            audio_16k = prepare_audio(raw)
+
+        model = _load_embed_model()
+        result = model.generate(input=audio_16k, output_dir=None)
+        raw_emb = result[0]["spk_embedding"]
+        if hasattr(raw_emb, "cpu"):
+            raw_emb = raw_emb.cpu().numpy()
+        emb = np.array(raw_emb).flatten()
+
+        # L2-normalize
+        norm = np.linalg.norm(emb)
+        if norm > 0:
+            emb = emb / norm
+
+        return {"embedding": emb.tolist(), "dim": len(emb)}
+    except Exception as e:
+        return JSONResponse(status_code=500, content={"error": str(e)})
+
+
+@app.post("/transcribe")
+async def transcribe(
+    audio: UploadFile = File(...),
+    prompt: str = Form("Transcribe this audio."),
+):
+    try:
+        raw = await audio.read()
+        audio_16k = prepare_audio(raw)
+
+        model, processor = _load_voxtral()
+        inputs = processor(
+            audios=audio_16k,
+            sampling_rate=TARGET_SR,
+            text=prompt,
+            return_tensors="pt",
+        ).to("cuda")
+
+        output_ids = model.generate(**inputs, max_new_tokens=1024)
+        text = processor.batch_decode(output_ids, skip_special_tokens=True)[0]
+        text = _clean_voxtral_text(text)
+
+        return {"text": text}
+    except Exception as e:
+        logger.exception("Transcription failed")
+        return JSONResponse(status_code=500, content={"error": str(e)})
+
+
+@app.post("/transcribe/stream")
+async def transcribe_stream(
+    audio: UploadFile = File(...),
+    prompt: str = Form("Transcribe this audio."),
+):
+    try:
+        raw = await audio.read()
+        audio_16k = prepare_audio(raw)
+    except Exception as e:
+        logger.exception("Audio preparation failed")
+        return JSONResponse(status_code=500, content={"error": str(e)})
+
+    async def event_generator():
+        try:
+            from transformers import TextIteratorStreamer
+
+            model, processor = _load_voxtral()
+            inputs = processor(
+                audios=audio_16k,
+                sampling_rate=TARGET_SR,
+                text=prompt,
+                return_tensors="pt",
+            ).to("cuda")
+
+            streamer = TextIteratorStreamer(
+                processor.tokenizer, skip_prompt=True, skip_special_tokens=True
+            )
+            gen_kwargs = {**inputs, "max_new_tokens": 1024, "streamer": streamer}
+
+            thread = threading.Thread(target=model.generate, kwargs=gen_kwargs)
+            thread.start()
+
+            full_text = ""
+            for chunk in streamer:
+                chunk = _MARKER_RE.sub("", chunk)
+                if chunk:
+                    full_text += chunk
+                    yield {"event": "token", "data": json.dumps({"token": chunk})}
+
+            thread.join()
+            full_text = " ".join(full_text.split()).strip()
+            yield {"event": "done", "data": json.dumps({"text": full_text})}
+        except Exception as e:
+            logger.exception("Streaming transcription failed")
+            yield {"event": "error", "data": json.dumps({"error": str(e)})}
+
+    return EventSourceResponse(event_generator())

requirements.txt

@@ -1,12 +1,13 @@
-fastapi>=0.115.0
-uvicorn[standard]>=0.30.0
-numpy>=1.26.0
-soundfile>=0.12.0
-librosa>=0.10.0
-pyannote.audio>=3.3.0
-funasr>=1.3.0
-python-multipart>=0.0.9
-pydub>=0.25.0
-transformers>=4.45.0
-accelerate>=0.34.0
-sse-starlette>=1.0.0
+fastapi>=0.115.0
+uvicorn[standard]>=0.30.0
+numpy>=1.26.0
+soundfile>=0.12.0
+librosa>=0.10.0
+pyannote.audio>=3.3.0
+funasr>=1.3.0
+python-multipart>=0.0.9
+pydub>=0.25.0
+transformers>=4.45.0
+accelerate>=0.34.0
+sse-starlette>=1.0.0
+torchvision>=0.19.0