|
|
""" |
|
|
realtime_server.py - Optional WebSocket server for real-time audio streaming |
|
|
Chạy riêng biệt: uvicorn realtime_server:app --host 0.0.0.0 --port 8000 |
|
|
""" |
|
|
|
|
|
import os |
|
|
import asyncio |
|
|
import json |
|
|
import base64 |
|
|
from typing import Optional |
|
|
from fastapi import FastAPI, WebSocket, WebSocketDisconnect |
|
|
from fastapi.responses import HTMLResponse |
|
|
import websockets |
|
|
import sys |
|
|
sys.dont_write_bytecode = True |
|
|
|
|
|
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "") |
|
|
OPENAI_REALTIME_MODEL = os.getenv("OPENAI_REALTIME_MODEL", "gpt-4o-realtime-preview") |
|
|
|
|
|
app = FastAPI() |
|
|
|
|
|
|
|
|
html = """ |
|
|
<!DOCTYPE html> |
|
|
<html> |
|
|
<head> |
|
|
<title>Realtime Audio Test</title> |
|
|
</head> |
|
|
<body> |
|
|
<h1>Realtime Audio Test</h1> |
|
|
<button id="startBtn">Start Recording</button> |
|
|
<button id="stopBtn" disabled>Stop Recording</button> |
|
|
<div id="status">Status: Ready</div> |
|
|
<div><label>Instructions: <input id="instructions" placeholder="Optional prompt" /></label></div> |
|
|
<div id="transcript"></div> |
|
|
<audio id="player" controls></audio> |
|
|
|
|
|
<script> |
|
|
let mediaRecorder; |
|
|
let audioChunks = []; |
|
|
let ws; |
|
|
|
|
|
function ensureWS() { |
|
|
if (ws && ws.readyState === WebSocket.OPEN) return ws; |
|
|
const basePath = (location.pathname.endsWith('/') ? location.pathname.slice(0,-1) : location.pathname); |
|
|
ws = new WebSocket((location.protocol === 'https:' ? 'wss://' : 'ws://') + location.host + basePath + '/ws'); |
|
|
ws.onopen = () => { |
|
|
document.getElementById('status').textContent = 'Status: WS connected'; |
|
|
}; |
|
|
ws.onmessage = (event) => { |
|
|
try { |
|
|
const msg = JSON.parse(event.data); |
|
|
if (msg.type === 'transcript_delta') { |
|
|
const el = document.getElementById('transcript'); |
|
|
el.innerHTML = `<strong>Transcript:</strong> ${el.textContent}${msg.text}`; |
|
|
} else if (msg.type === 'response_completed') { |
|
|
if (msg.audio) { |
|
|
const b64 = msg.audio; |
|
|
const audioBlob = base64ToWavBlob(b64); |
|
|
const url = URL.createObjectURL(audioBlob); |
|
|
const player = document.getElementById('player'); |
|
|
player.src = url; |
|
|
player.play(); |
|
|
} |
|
|
document.getElementById('status').textContent = 'Status: Completed'; |
|
|
} |
|
|
} catch {} |
|
|
}; |
|
|
ws.onclose = () => { |
|
|
document.getElementById('status').textContent = 'Status: WS closed'; |
|
|
}; |
|
|
return ws; |
|
|
} |
|
|
|
|
|
function base64ToWavBlob(base64) { |
|
|
const byteCharacters = atob(base64); |
|
|
const byteNumbers = new Array(byteCharacters.length); |
|
|
for (let i = 0; i < byteCharacters.length; i++) { |
|
|
byteNumbers[i] = byteCharacters.charCodeAt(i); |
|
|
} |
|
|
const byteArray = new Uint8Array(byteNumbers); |
|
|
return new Blob([byteArray], { type: 'audio/wav' }); |
|
|
} |
|
|
|
|
|
document.getElementById('startBtn').onclick = async () => { |
|
|
ensureWS(); |
|
|
const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); |
|
|
mediaRecorder = new MediaRecorder(stream); |
|
|
|
|
|
audioChunks = []; |
|
|
document.getElementById('transcript').textContent = ''; |
|
|
|
|
|
mediaRecorder.ondataavailable = (event) => { |
|
|
audioChunks.push(event.data); |
|
|
}; |
|
|
|
|
|
mediaRecorder.onstop = async () => { |
|
|
const audioBlob = new Blob(audioChunks, { type: 'audio/wav' }); |
|
|
audioChunks = []; |
|
|
const reader = new FileReader(); |
|
|
reader.readAsDataURL(audioBlob); |
|
|
reader.onloadend = () => { |
|
|
const base64data = reader.result.split(',')[1]; |
|
|
const instructions = document.getElementById('instructions').value || ''; |
|
|
ws.send(JSON.stringify({ type: 'utterance', audio: base64data, instructions })); |
|
|
document.getElementById('status').textContent = 'Status: Sending to OpenAI...'; |
|
|
}; |
|
|
}; |
|
|
|
|
|
mediaRecorder.start(); |
|
|
document.getElementById('startBtn').disabled = true; |
|
|
document.getElementById('stopBtn').disabled = false; |
|
|
document.getElementById('status').textContent = 'Status: Recording...'; |
|
|
}; |
|
|
|
|
|
document.getElementById('stopBtn').onclick = () => { |
|
|
mediaRecorder.stop(); |
|
|
document.getElementById('startBtn').disabled = false; |
|
|
document.getElementById('stopBtn').disabled = true; |
|
|
}; |
|
|
</script> |
|
|
</body> |
|
|
</html> |
|
|
""" |
|
|
|
|
|
@app.get("/") |
|
|
async def get(): |
|
|
return HTMLResponse(html) |
|
|
|
|
|
@app.post("/process-audio") |
|
|
async def process_audio(request: dict): |
|
|
"""Process audio from frontend""" |
|
|
try: |
|
|
audio_data = base64.b64decode(request.get("audio", "")) |
|
|
|
|
|
|
|
|
import tempfile |
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as f: |
|
|
f.write(audio_data) |
|
|
temp_path = f.name |
|
|
|
|
|
|
|
|
from openai import OpenAI |
|
|
client = OpenAI(api_key=OPENAI_API_KEY) |
|
|
|
|
|
with open(temp_path, "rb") as audio_file: |
|
|
transcript = client.audio.transcriptions.create( |
|
|
model="whisper-1", |
|
|
file=audio_file, |
|
|
language="de" |
|
|
) |
|
|
|
|
|
|
|
|
import os |
|
|
os.unlink(temp_path) |
|
|
|
|
|
return {"success": True, "transcript": transcript.text} |
|
|
|
|
|
except Exception as e: |
|
|
return {"success": False, "error": str(e)} |
|
|
|
|
|
@app.websocket("/ws") |
|
|
async def websocket_endpoint(websocket: WebSocket): |
|
|
"""WebSocket endpoint for real-time audio streaming""" |
|
|
await websocket.accept() |
|
|
|
|
|
try: |
|
|
|
|
|
headers = { |
|
|
"Authorization": f"Bearer {OPENAI_API_KEY}", |
|
|
"OpenAI-Beta": "realtime=v1", |
|
|
} |
|
|
|
|
|
async with websockets.connect( |
|
|
f"wss://api.openai.com/v1/realtime?model={OPENAI_REALTIME_MODEL}", |
|
|
extra_headers=headers |
|
|
) as openai_ws: |
|
|
|
|
|
async def process_utterance(b64_wav: str, instructions: Optional[str] = None): |
|
|
|
|
|
await openai_ws.send(json.dumps({ |
|
|
"type": "input_audio_buffer.append", |
|
|
"audio": {"data": b64_wav, "format": "wav"} |
|
|
})) |
|
|
|
|
|
await openai_ws.send(json.dumps({ |
|
|
"type": "input_audio_buffer.commit" |
|
|
})) |
|
|
|
|
|
await openai_ws.send(json.dumps({ |
|
|
"type": "response.create", |
|
|
"response": { |
|
|
"modalities": ["audio", "text"], |
|
|
"instructions": instructions or "" |
|
|
} |
|
|
})) |
|
|
|
|
|
audio_chunks = [] |
|
|
transcript = "" |
|
|
|
|
|
while True: |
|
|
msg = await openai_ws.recv() |
|
|
try: |
|
|
event = json.loads(msg) |
|
|
except: |
|
|
continue |
|
|
|
|
|
etype = event.get("type") |
|
|
if etype == "response.audio.delta": |
|
|
data = event.get("delta") or event.get("data") |
|
|
if data: |
|
|
audio_chunks.append(data) |
|
|
await websocket.send_text(json.dumps({ |
|
|
"type": "audio_delta", |
|
|
"data": data |
|
|
})) |
|
|
elif etype == "response.transcript.delta": |
|
|
delta = event.get("delta", "") |
|
|
transcript += delta |
|
|
await websocket.send_text(json.dumps({ |
|
|
"type": "transcript_delta", |
|
|
"text": delta |
|
|
})) |
|
|
elif etype == "response.completed": |
|
|
await websocket.send_text(json.dumps({ |
|
|
"type": "response_completed", |
|
|
"transcript": transcript, |
|
|
"audio": "".join(audio_chunks) |
|
|
})) |
|
|
break |
|
|
|
|
|
|
|
|
while True: |
|
|
try: |
|
|
text = await websocket.receive_text() |
|
|
except WebSocketDisconnect: |
|
|
break |
|
|
try: |
|
|
msg = json.loads(text) |
|
|
except: |
|
|
continue |
|
|
|
|
|
mtype = msg.get("type") |
|
|
if mtype == "utterance": |
|
|
b64_wav = msg.get("audio", "") |
|
|
instructions = msg.get("instructions", "") |
|
|
if b64_wav: |
|
|
await process_utterance(b64_wav, instructions) |
|
|
elif mtype == "ping": |
|
|
await websocket.send_text(json.dumps({"type": "pong"})) |
|
|
|
|
|
except Exception as e: |
|
|
print(f"WebSocket error: {e}") |
|
|
finally: |
|
|
await websocket.close() |
|
|
|
|
|
if __name__ == "__main__": |
|
|
import uvicorn |
|
|
uvicorn.run(app, host="0.0.0.0", port=8000) |
|
|
|
