server.py

import numpy as np
import torch
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from faster_whisper import WhisperModel

app = FastAPI()

# Load ASR model using faster-whisper from local directory
print("Loading Whisper model from local whisper-small-yor-ct2...")
# Select device and compute type dynamically
device = "cuda" if torch.cuda.is_available() else "cpu"
compute_type = "float16" if torch.cuda.is_available() else "int8"

asr_model = WhisperModel(
    "whisper-small-yor-ct2",
    device=device,
    compute_type=compute_type
)
print("ASR model loaded successfully.")

from silero_vad import VADIterator, load_silero_vad

# Load Silero VAD model
print("Loading Silero VAD model...")
vad_model = load_silero_vad()
print("VAD model loaded successfully.")

@app.get("/")
def read_root():
    return {"message": "Yoruba Real-time ASR Server is running."}

@app.websocket("/stream")
async def websocket_endpoint(websocket: WebSocket):
    await websocket.accept()
    print("WebSocket client connected.")
    
    # Initialize VAD iterator with a sample rate of 16000 and 0.6 threshold for noise robustness
    vad_iterator = VADIterator(vad_model, threshold=0.6, sampling_rate=16000)
    
    # We will accumulate audio samples for transcription
    speech_buffer = []
    accumulator = np.array([], dtype=np.float32)
    
    try:
        while True:
            # Receive audio bytes (16 kHz PCM 16-bit mono)
            data = await websocket.receive_bytes()
            if not data:
                break
                
            # Convert bytes to numpy int16, then float32 scaled to [-1.0, 1.0]
            audio_chunk = np.frombuffer(data, dtype=np.int16).astype(np.float32) / 32768.0
            
            # Append new chunk to the accumulator
            accumulator = np.concatenate((accumulator, audio_chunk))
            
            # Process in blocks of exactly 512 samples (Silero VAD requirement)
            chunk_size = 512
            while len(accumulator) >= chunk_size:
                sub_chunk = accumulator[:chunk_size]
                accumulator = accumulator[chunk_size:]
                
                tensor_chunk = torch.from_numpy(sub_chunk)
                speech_dict = vad_iterator(tensor_chunk, return_seconds=True)
                
                # If speech is detected, accumulate the frames
                if vad_iterator.triggered:
                    speech_buffer.append(sub_chunk)
                
                # Force transcription if a single speech segment exceeds ~5.0 seconds
                # 5 seconds @ 16kHz is 80000 samples. 156 chunks * 512 = 79872 samples (~5.0s).
                force_transcribe = len(speech_buffer) >= 156
                
                # When speech ends, or we force it due to length
                if (speech_dict and "end" in speech_dict) or force_transcribe:
                    if speech_buffer:
                        full_audio = np.concatenate(speech_buffer)
                        # Clear buffer
                        speech_buffer = []
                        
                        print(f"Transcribing {len(full_audio)/16000:.2f}s of speech (forced={force_transcribe})...")
                        # Transcribe using faster-whisper
                        segments, info = asr_model.transcribe(
                            full_audio, 
                            language="yo", 
                            beam_size=2,
                            condition_on_previous_text=False,
                            no_repeat_ngram_size=3
                        )
                        text = " ".join([segment.text for segment in segments]).strip()
                        
                        if text:
                            print(f"Result: {text}")
                            await websocket.send_json({
                                "status": "final",
                                "text": text
                            })
                            
                    vad_iterator.reset_states()
                    
    except WebSocketDisconnect:
        print("WebSocket client disconnected.")
    except Exception as e:
        print(f"Error in stream processing: {e}")
    finally:
        # Transcribe any remaining audio in the buffer at disconnection
        # If we are currently in a speech segment, add remaining accumulator samples
        if len(accumulator) > 0 and vad_iterator.triggered:
            speech_buffer.append(accumulator)
            
        if speech_buffer:
            try:
                full_audio = np.concatenate(speech_buffer)
                print(f"Transcribing remaining {len(full_audio)/16000:.2f}s of speech...")
                segments, info = asr_model.transcribe(
                    full_audio, 
                    language="yo", 
                    beam_size=2,
                    condition_on_previous_text=False,
                    no_repeat_ngram_size=3
                )
                text = " ".join([segment.text for segment in segments]).strip()
                if text:
                    print(f"Final Result: {text}")
                    await websocket.send_json({
                        "status": "final",
                        "text": text
                    })
            except Exception as e:
                print(f"Error sending final transcript: {e}")
        try:
            await websocket.close()
        except Exception:
            pass
        print("WebSocket connection cleaned up.")