from __future__ import annotations import asyncio import base64 import io import json import wave from pathlib import Path from typing import Any from fastapi import WebSocket, WebSocketDisconnect from pydantic import BaseModel from time_machine.application.container import AppContainer from time_machine.domain.events import ( AudioEvent, ConversationTextEvent, ErrorEvent, TemporalEvent, TranscriptEvent, ) DEFAULT_SAMPLE_RATE = 16000 DEFAULT_CHANNELS = 1 MAX_TURN_AUDIO_BYTES = 16 * 1024 * 1024 async def realtime_voice_socket(websocket: WebSocket, container: AppContainer) -> None: await websocket.accept() state = _RealtimeConnectionState() await _send_json(websocket, {"type": "ready", "sample_rate": DEFAULT_SAMPLE_RATE}) try: while True: message = await websocket.receive() if message.get("type") == "websocket.disconnect": return if "text" in message and message["text"] is not None: await _handle_control_message(websocket, container, state, message["text"]) elif "bytes" in message and message["bytes"] is not None: state.append_audio(message["bytes"]) except WebSocketDisconnect: return async def _handle_control_message( websocket: WebSocket, container: AppContainer, state: "_RealtimeConnectionState", raw_message: str, ) -> None: try: message = json.loads(raw_message) except json.JSONDecodeError: await _send_json(websocket, {"type": "error", "message": "Invalid realtime message."}) return message_type = message.get("type") if message_type == "start": encounter_id = _clean_string(message.get("encounter_id")) if not encounter_id: await _send_json(websocket, {"type": "error", "message": "Launch the machine first."}) return state.encounter_id = encounter_id state.sample_rate = _positive_int(message.get("sample_rate"), DEFAULT_SAMPLE_RATE) state.channels = _positive_int(message.get("channels"), DEFAULT_CHANNELS) state.reset_audio() await _send_json(websocket, {"type": "listening", "encounter_id": encounter_id}) return if message_type == "cancel": state.reset_audio() await _send_json(websocket, {"type": "listening", "encounter_id": state.encounter_id}) return if message_type == "commit": if not state.encounter_id: await _send_json(websocket, {"type": "error", "message": "Launch the machine first."}) state.reset_audio() return audio = state.consume_wav() if not audio: await _send_json(websocket, {"type": "listening", "encounter_id": state.encounter_id}) return await _send_json(websocket, {"type": "processing", "message": "Decoding signal..."}) try: await _stream_audio_turn(websocket, container, state.encounter_id, audio) except Exception as exc: await _send_json(websocket, {"type": "error", "message": str(exc)}) await _send_json(websocket, {"type": "listening", "encounter_id": state.encounter_id}) return await _send_json(websocket, {"type": "error", "message": f"Unknown realtime message: {message_type}"}) async def _stream_audio_turn( websocket: WebSocket, container: AppContainer, encounter_id: str, audio: bytes, ) -> None: iterator = container.speech_orchestrator.handle_audio_turn( encounter_id=encounter_id, audio=audio, ) sentinel = object() while True: event = await asyncio.to_thread(next, iterator, sentinel) if event is sentinel: return await _send_event(websocket, event) async def _send_event(websocket: WebSocket, event: TemporalEvent) -> None: message = _event_payload(event) await _send_json(websocket, message) def _event_payload(event: TemporalEvent) -> dict[str, Any]: payload = { "type": "event", "event_type": event.event_type, "message": event.message, "metadata": event.metadata, } if isinstance(event, TranscriptEvent): payload.update( { "type": "transcript", "text": event.transcript.text, "is_final": event.transcript.is_final, "confidence": event.transcript.confidence, "language": event.transcript.language, } ) elif isinstance(event, ConversationTextEvent): payload.update( { "type": "conversation_text", "text": event.text, "is_final": event.is_final, } ) elif isinstance(event, AudioEvent): payload.update(_audio_payload(event)) elif isinstance(event, ErrorEvent): payload.update({"type": "error", "severity": event.severity}) else: serialized = event.model_dump(mode="json") serialized.pop("created_at", None) payload["event"] = serialized return payload def _audio_payload(event: AudioEvent) -> dict[str, Any]: audio = event.audio path = Path(audio.path) if audio.path else None if path and path.exists(): audio_b64 = base64.b64encode(path.read_bytes()).decode("ascii") else: audio_b64 = "" return { "type": "audio", "audio_b64": audio_b64, "mime_type": audio.mime_type, "duration_seconds": audio.duration_seconds, "description": audio.description, } async def _send_json(websocket: WebSocket, payload: dict[str, Any]) -> None: await websocket.send_text(json.dumps(payload, ensure_ascii=True)) class _RealtimeConnectionState(BaseModel): encounter_id: str | None = None sample_rate: int = DEFAULT_SAMPLE_RATE channels: int = DEFAULT_CHANNELS audio_chunks: list[bytes] = [] audio_bytes: int = 0 def append_audio(self, chunk: bytes) -> None: if not chunk: return if self.audio_bytes + len(chunk) > MAX_TURN_AUDIO_BYTES: self.reset_audio() return self.audio_chunks.append(chunk) self.audio_bytes += len(chunk) def consume_wav(self) -> bytes: if not self.audio_chunks: return b"" pcm = b"".join(self.audio_chunks) self.reset_audio() if not pcm: return b"" return _pcm16_to_wav(pcm, self.sample_rate, self.channels) def reset_audio(self) -> None: self.audio_chunks = [] self.audio_bytes = 0 def _pcm16_to_wav(pcm: bytes, sample_rate: int, channels: int) -> bytes: output = io.BytesIO() with wave.open(output, "wb") as handle: handle.setnchannels(channels) handle.setsampwidth(2) handle.setframerate(sample_rate) handle.writeframes(pcm) return output.getvalue() def _clean_string(value: object) -> str | None: return value.strip() if isinstance(value, str) and value.strip() else None def _positive_int(value: object, default: int) -> int: try: parsed = int(value) except (TypeError, ValueError): return default return parsed if parsed > 0 else default