Upload folder using huggingface_hub

README.md

@@ -9,6 +9,7 @@ app_file: app.py
 pinned: false
 license: mit
 short_description: Compare Round Trip Times between WebRTC and Websockets
+tags: [webrtc, websocket, gradio, secret|TWILIO_ACCOUNT_SID, secret|TWILIO_AUTH_TOKEN, secret|ELEVENLABS_API_KEY, secret|GROQ_API_KEY, secret|ANTHROPIC_API_KEY]
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,118 @@
+from dotenv import load_dotenv
+
+import numpy as np
+import gradio as gr
+from gradio.utils import get_space
+from fastrtc import ReplyOnPause, Stream, AdditionalOutputs, get_twilio_turn_credentials
+from fastrtc.utils import audio_to_bytes, aggregate_bytes_to_16bit
+from pathlib import Path
+from fastapi.responses import HTMLResponse, StreamingResponse
+from groq import Groq
+import anthropic
+from elevenlabs import ElevenLabs
+import os
+from pydantic import BaseModel
+import json
+
+
+load_dotenv()
+
+groq_client = Groq()
+claude_client = anthropic.Anthropic()
+tts_client = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"])
+
+curr_dir = Path(__file__).parent
+
+
+def response(
+    audio: tuple[int, np.ndarray],
+    chatbot: list[dict] | None = None,
+):
+    chatbot = chatbot or []
+    messages = [{"role": d["role"], "content": d["content"]} for d in chatbot]
+    prompt = groq_client.audio.transcriptions.create(
+        file=("audio-file.mp3", audio_to_bytes(audio)),
+        model="whisper-large-v3-turbo",
+        response_format="verbose_json",
+    ).text
+    print("prompt", prompt)
+    chatbot.append({"role": "user", "content": prompt})
+    messages.append({"role": "user", "content": prompt})
+    response = claude_client.messages.create(
+        model="claude-3-5-haiku-20241022",
+        max_tokens=512,
+        messages=messages,  # type: ignore
+    )
+    response_text = " ".join(
+        block.text  # type: ignore
+        for block in response.content
+        if getattr(block, "type", None) == "text"
+    )
+    chatbot.append({"role": "assistant", "content": response_text})
+    yield AdditionalOutputs(chatbot)
+    iterator = tts_client.text_to_speech.convert_as_stream(
+        text=response_text,
+        voice_id="JBFqnCBsd6RMkjVDRZzb",
+        model_id="eleven_multilingual_v2",
+        output_format="pcm_24000",
+    )
+    for chunk in aggregate_bytes_to_16bit(iterator):
+        audio_array = np.frombuffer(chunk, dtype=np.int16).reshape(1, -1)
+        yield (24000, audio_array, "mono")
+
+
+chatbot = gr.Chatbot(type="messages")
+stream = Stream(
+    modality="audio",
+    mode="send-receive",
+    handler=ReplyOnPause(response),
+    additional_outputs_handler=lambda a, b: b,
+    additional_inputs=[chatbot],
+    additional_outputs=[chatbot],
+    rtc_configuration=get_twilio_turn_credentials()
+    if get_space() == "gradio"
+    else None,
+)
+
+
+class Message(BaseModel):
+    role: str
+    content: str
+
+
+class InputData(BaseModel):
+    webrtc_id: str
+    chatbot: list[Message]
+
+
+@stream.get("/")
+async def _():
+    rtc_config = get_twilio_turn_credentials() if get_space() else "{}"
+    html_content = (curr_dir / "index.html").read_text()
+    html_content = html_content.replace("__RTC_CONFIGURATION__", json.dumps(rtc_config))
+    return HTMLResponse(content=html_content, status_code=200)
+
+
+@stream.post("/input_hook")
+async def _(body: InputData):
+    stream.set_input(body.webrtc_id, body.model_dump()["chatbot"])
+    return {"status": "ok"}
+
+
+@stream.get("/outputs")
+def _(webrtc_id: str):
+    print("outputs", webrtc_id)
+
+    async def output_stream():
+        async for output in stream.output_stream(webrtc_id):
+            chatbot = output.args[0]
+            yield f"event: output\ndata: {json.dumps(chatbot[-2])}\n\n"
+            yield f"event: output\ndata: {json.dumps(chatbot[-1])}\n\n"
+
+    return StreamingResponse(output_stream(), media_type="text/event-stream")
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    s = uvicorn.run(stream, port=7860, host="0.0.0.0")

index.html

@@ -0,0 +1,569 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>WebRTC vs WebSocket Benchmark</title>
+    <script src="https://cdn.jsdelivr.net/npm/alawmulaw"></script>
+    <style>
+        body {
+            font-family: system-ui, -apple-system, sans-serif;
+            margin: 0;
+            padding: 20px;
+            background-color: #f5f5f5;
+        }
+
+        .container {
+            display: grid;
+            grid-template-columns: 1fr 1fr;
+            gap: 30px;
+            max-width: 1400px;
+            margin: 0 auto;
+        }
+
+        .panel {
+            background: white;
+            border-radius: 12px;
+            padding: 20px;
+            box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+        }
+
+        .chat-container {
+            height: 400px;
+            overflow-y: auto;
+            border: 1px solid #e0e0e0;
+            border-radius: 8px;
+            padding: 15px;
+            margin-bottom: 15px;
+        }
+
+        .message {
+            margin-bottom: 10px;
+            padding: 8px 12px;
+            border-radius: 8px;
+            max-width: 80%;
+        }
+
+        .message.user {
+            background-color: #e3f2fd;
+            margin-left: auto;
+        }
+
+        .message.assistant {
+            background-color: #f5f5f5;
+        }
+
+        .metrics {
+            margin-top: 15px;
+            padding: 10px;
+            background: #f8f9fa;
+            border-radius: 8px;
+        }
+
+        .metric {
+            margin: 5px 0;
+            font-size: 14px;
+        }
+
+        button {
+            background-color: #1976d2;
+            color: white;
+            border: none;
+            padding: 10px 20px;
+            border-radius: 6px;
+            cursor: pointer;
+            font-size: 14px;
+            transition: background-color 0.2s;
+        }
+
+        button:hover {
+            background-color: #1565c0;
+        }
+
+        button:disabled {
+            background-color: #bdbdbd;
+            cursor: not-allowed;
+        }
+
+        h2 {
+            margin-top: 0;
+            color: #1976d2;
+        }
+
+        .visualizer {
+            width: 100%;
+            height: 100px;
+            margin: 10px 0;
+            background: #fafafa;
+            border-radius: 8px;
+        }
+
+        /* Add styles for disclaimer */
+        .disclaimer {
+            background-color: #fff3e0;
+            padding: 15px;
+            border-radius: 8px;
+            margin-bottom: 20px;
+            font-size: 14px;
+            line-height: 1.5;
+            max-width: 1400px;
+            margin: 0 auto 20px auto;
+        }
+
+        /* Update nav bar styles */
+        .nav-bar {
+            background-color: #f5f5f5;
+            padding: 10px 20px;
+            margin-bottom: 20px;
+        }
+
+        .nav-container {
+            max-width: 1400px;
+            margin: 0 auto;
+            display: flex;
+            gap: 10px;
+        }
+
+        .nav-button {
+            background-color: #1976d2;
+            color: white;
+            border: none;
+            padding: 8px 16px;
+            border-radius: 4px;
+            cursor: pointer;
+            text-decoration: none;
+            font-size: 14px;
+            transition: background-color 0.2s;
+        }
+
+        .nav-button:hover {
+            background-color: #1565c0;
+        }
+    </style>
+</head>
+
+<body>
+    <nav class="nav-bar">
+        <div class="nav-container">
+            <a href="/webrtc/docs" class="nav-button">WebRTC Docs</a>
+            <a href="/websocket/docs" class="nav-button">WebSocket Docs</a>
+            <a href="/telephone/docs" class="nav-button">Telephone Docs</a>
+            <a href="/ui" class="nav-button">UI</a>
+        </div>
+    </nav>
+
+    <div class="disclaimer">
+        This page compares the WebRTC Round-Trip-Time calculated from <code>getStats()</code> to the time taken to
+        process a ping/pong response pattern over websockets. It may not be a gold standard benchmark. Both WebRTC and
+        Websockets have their merits/advantages which is why FastRTC supports both. Artifacts in the WebSocket playback
+        audio are due to gaps in my frontend processing code and not FastRTC web server.
+    </div>
+
+    <div class="container">
+        <div class="panel">
+            <h2>WebRTC Connection</h2>
+            <div id="webrtc-chat" class="chat-container"></div>
+            <div id="webrtc-metrics" class="metrics">
+                <div class="metric">RTT (Round Trip Time): <span id="webrtc-rtt">-</span></div>
+            </div>
+            <button id="webrtc-button">Connect WebRTC</button>
+        </div>
+
+        <div class="panel">
+            <h2>WebSocket Connection</h2>
+            <div id="ws-chat" class="chat-container"></div>
+            <div id="ws-metrics" class="metrics">
+                <div class="metric">RTT (Round Trip Time): <span id="ws-rtt">0</span></div>
+            </div>
+            <button id="ws-button">Connect WebSocket</button>
+        </div>
+    </div>
+
+    <audio id="webrtc-audio" style="display: none;"></audio>
+    <audio id="ws-audio" style="display: none;"></audio>
+
+    <script>
+        // Shared utilities
+        function generateId() {
+            return Math.random().toString(36).substring(7);
+        }
+
+        function sendInput(id) {
+
+            return function handleMessage(event) {
+                const eventJson = JSON.parse(event.data);
+                if (eventJson.type === "send_input") {
+                    fetch('/input_hook', {
+                        method: 'POST',
+                        headers: {
+                            'Content-Type': 'application/json',
+                        },
+                        body: JSON.stringify({
+                            webrtc_id: id,
+                            chatbot: chatHistoryWebRTC
+                        })
+                    });
+                }
+            }
+        }
+
+        let chatHistoryWebRTC = [];
+        let chatHistoryWebSocket = [];
+
+        function addMessage(containerId, role, content) {
+            const container = document.getElementById(containerId);
+            const messageDiv = document.createElement('div');
+            messageDiv.classList.add('message', role);
+            messageDiv.textContent = content;
+            container.appendChild(messageDiv);
+            container.scrollTop = container.scrollHeight;
+            if (containerId === 'webrtc-chat') {
+                chatHistoryWebRTC.push({ role, content });
+            } else {
+                chatHistoryWebSocket.push({ role, content });
+            }
+        }
+
+        // WebRTC Implementation
+        let webrtcPeerConnection;
+
+        // Add this function to collect RTT stats
+        async function updateWebRTCStats() {
+            if (!webrtcPeerConnection) return;
+
+            const stats = await webrtcPeerConnection.getStats();
+            stats.forEach(report => {
+                if (report.type === 'candidate-pair' && report.state === 'succeeded') {
+                    const rtt = report.currentRoundTripTime * 1000; // Convert to ms
+                    document.getElementById('webrtc-rtt').textContent = `${rtt.toFixed(2)}ms`;
+                }
+            });
+        }
+
+        async function setupWebRTC() {
+            const button = document.getElementById('webrtc-button');
+            button.textContent = "Stop";
+
+            const config = __RTC_CONFIGURATION__;
+            webrtcPeerConnection = new RTCPeerConnection(config);
+            const webrtcId = generateId();
+
+            try {
+                const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+                stream.getTracks().forEach(track => {
+                    webrtcPeerConnection.addTrack(track, stream);
+                });
+
+                webrtcPeerConnection.addEventListener('track', (evt) => {
+                    const audio = document.getElementById('webrtc-audio');
+                    if (audio.srcObject !== evt.streams[0]) {
+                        audio.srcObject = evt.streams[0];
+                        audio.play();
+                    }
+                });
+
+                const dataChannel = webrtcPeerConnection.createDataChannel('text');
+                dataChannel.onmessage = sendInput(webrtcId);
+
+                const offer = await webrtcPeerConnection.createOffer();
+                await webrtcPeerConnection.setLocalDescription(offer);
+
+                const response = await fetch('/webrtc/offer', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({
+                        sdp: offer.sdp,
+                        type: offer.type,
+                        webrtc_id: webrtcId
+                    })
+                });
+
+                const serverResponse = await response.json();
+                await webrtcPeerConnection.setRemoteDescription(serverResponse);
+
+                // Setup event source for messages
+                const eventSource = new EventSource('/outputs?webrtc_id=' + webrtcId);
+                eventSource.addEventListener("output", (event) => {
+                    const eventJson = JSON.parse(event.data);
+                    addMessage('webrtc-chat', eventJson.role, eventJson.content);
+                });
+
+                // Add periodic stats collection
+                const statsInterval = setInterval(updateWebRTCStats, 1000);
+
+                // Store the interval ID on the connection
+                webrtcPeerConnection.statsInterval = statsInterval;
+
+            } catch (err) {
+                console.error('WebRTC setup error:', err);
+            }
+        }
+
+        function webrtc_stop() {
+            if (webrtcPeerConnection) {
+                // Clear the stats interval
+                if (webrtcPeerConnection.statsInterval) {
+                    clearInterval(webrtcPeerConnection.statsInterval);
+                }
+
+                // Close all tracks
+                webrtcPeerConnection.getSenders().forEach(sender => {
+                    if (sender.track) {
+                        sender.track.stop();
+                    }
+                });
+
+                webrtcPeerConnection.close();
+                webrtcPeerConnection = null;
+
+                // Reset metrics display
+                document.getElementById('webrtc-rtt').textContent = '-';
+            }
+        }
+
+        // WebSocket Implementation
+        let webSocket;
+        let wsMetrics = {
+            pingStartTime: 0,
+            rttValues: []
+        };
+
+        // Load mu-law library
+
+        // Add load promise to track when the script is ready
+
+
+        function resample(audioData, fromSampleRate, toSampleRate) {
+            const ratio = fromSampleRate / toSampleRate;
+            const newLength = Math.round(audioData.length / ratio);
+            const result = new Float32Array(newLength);
+
+            for (let i = 0; i < newLength; i++) {
+                const position = i * ratio;
+                const index = Math.floor(position);
+                const fraction = position - index;
+
+                if (index + 1 < audioData.length) {
+                    result[i] = audioData[index] * (1 - fraction) + audioData[index + 1] * fraction;
+                } else {
+                    result[i] = audioData[index];
+                }
+            }
+            return result;
+        }
+
+        function convertToMulaw(audioData, sampleRate) {
+            // Resample to 8000 Hz if needed
+            if (sampleRate !== 8000) {
+                audioData = resample(audioData, sampleRate, 8000);
+            }
+
+            // Convert float32 [-1,1] to int16 [-32768,32767]
+            const int16Data = new Int16Array(audioData.length);
+            for (let i = 0; i < audioData.length; i++) {
+                int16Data[i] = Math.floor(audioData[i] * 32768);
+            }
+
+            // Convert to mu-law using the library
+            return alawmulaw.mulaw.encode(int16Data);
+        }
+
+        async function setupWebSocket() {
+            const button = document.getElementById('ws-button');
+            button.textContent = "Stop";
+
+            try {
+                const stream = await navigator.mediaDevices.getUserMedia({
+                    audio: {
+                        "echoCancellation": true,
+                        "noiseSuppression": { "exact": true },
+                        "autoGainControl": { "exact": true },
+                        "sampleRate": { "ideal": 24000 },
+                        "sampleSize": { "ideal": 16 },
+                        "channelCount": { "exact": 1 },
+                    }
+                });
+                const wsId = generateId();
+                wsMetrics.startTime = performance.now();
+
+                // Create audio context and analyser for visualization
+                const audioContext = new AudioContext();
+                const analyser = audioContext.createAnalyser();
+                const source = audioContext.createMediaStreamSource(stream);
+                source.connect(analyser);
+
+                // Connect to websocket endpoint
+                webSocket = new WebSocket(`${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/websocket/offer`);
+
+                webSocket.onopen = () => {
+                    // Send initial start message
+                    webSocket.send(JSON.stringify({
+                        event: "start",
+                        websocket_id: wsId
+                    }));
+
+                    // Setup audio processing
+                    const processor = audioContext.createScriptProcessor(2048, 1, 1);
+                    source.connect(processor);
+                    processor.connect(audioContext.destination);
+
+                    processor.onaudioprocess = (e) => {
+                        const inputData = e.inputBuffer.getChannelData(0);
+                        const mulawData = convertToMulaw(inputData, audioContext.sampleRate);
+                        const base64Audio = btoa(String.fromCharCode.apply(null, mulawData));
+                        if (webSocket.readyState === WebSocket.OPEN) {
+                            webSocket.send(JSON.stringify({
+                                event: "media",
+                                media: {
+                                    payload: base64Audio
+                                }
+                            }));
+                        }
+                    };
+
+                    // Add ping interval
+                    webSocket.pingInterval = setInterval(() => {
+                        wsMetrics.pingStartTime = performance.now();
+                        webSocket.send(JSON.stringify({
+                            event: "ping"
+                        }));
+                    }, 500);
+                };
+
+                // Setup audio output context
+                const outputContext = new AudioContext({ sampleRate: 24000 });
+                const sampleRate = 24000; // Updated to match server sample rate
+                let audioQueue = [];
+                let isPlaying = false;
+
+                webSocket.onmessage = (event) => {
+                    const data = JSON.parse(event.data);
+                    if (data?.type === "send_input") {
+                        console.log("sending input")
+                        fetch('/input_hook', {
+                            method: 'POST',
+                            headers: { 'Content-Type': 'application/json' },
+                            body: JSON.stringify({ webrtc_id: wsId, chatbot: chatHistoryWebSocket })
+                        });
+                    }
+                    if (data.event === "media") {
+                        // Process received audio
+                        const audioData = atob(data.media.payload);
+                        const mulawData = new Uint8Array(audioData.length);
+                        for (let i = 0; i < audioData.length; i++) {
+                            mulawData[i] = audioData.charCodeAt(i);
+                        }
+
+                        // Convert mu-law to linear PCM
+                        const linearData = alawmulaw.mulaw.decode(mulawData);
+
+                        // Create an AudioBuffer
+                        const audioBuffer = outputContext.createBuffer(1, linearData.length, sampleRate);
+                        const channelData = audioBuffer.getChannelData(0);
+
+                        // Fill the buffer with the decoded data
+                        for (let i = 0; i < linearData.length; i++) {
+                            channelData[i] = linearData[i] / 32768.0;
+                        }
+
+                        // Queue the audio buffer
+                        audioQueue.push(audioBuffer);
+
+                        // Start playing if not already playing
+                        if (!isPlaying) {
+                            playNextBuffer();
+                        }
+                    }
+
+                    // Add pong handler
+                    if (data.event === "pong") {
+                        const rtt = performance.now() - wsMetrics.pingStartTime;
+                        wsMetrics.rttValues.push(rtt);
+                        // Keep only last 20 values for running mean
+                        if (wsMetrics.rttValues.length > 20) {
+                            wsMetrics.rttValues.shift();
+                        }
+                        const avgRtt = wsMetrics.rttValues.reduce((a, b) => a + b, 0) / wsMetrics.rttValues.length;
+                        document.getElementById('ws-rtt').textContent = `${avgRtt.toFixed(2)}ms`;
+                        return;
+                    }
+                };
+
+                function playNextBuffer() {
+                    if (audioQueue.length === 0) {
+                        isPlaying = false;
+                        return;
+                    }
+
+                    isPlaying = true;
+                    const bufferSource = outputContext.createBufferSource();
+                    bufferSource.buffer = audioQueue.shift();
+                    bufferSource.connect(outputContext.destination);
+
+                    bufferSource.onended = playNextBuffer;
+                    bufferSource.start();
+                }
+
+                const eventSource = new EventSource('/outputs?webrtc_id=' + wsId);
+                eventSource.addEventListener("output", (event) => {
+                    console.log("ws output", event);
+                    const eventJson = JSON.parse(event.data);
+                    addMessage('ws-chat', eventJson.role, eventJson.content);
+                });
+
+            } catch (err) {
+                console.error('WebSocket setup error:', err);
+                button.disabled = false;
+            }
+        }
+
+        function ws_stop() {
+            if (webSocket) {
+                webSocket.send(JSON.stringify({
+                    event: "stop"
+                }));
+                // Clear ping interval
+                if (webSocket.pingInterval) {
+                    clearInterval(webSocket.pingInterval);
+                }
+                // Reset RTT display
+                document.getElementById('ws-rtt').textContent = '-';
+                wsMetrics.rttValues = [];
+
+                // Clear the stats interval
+                if (webSocket.statsInterval) {
+                    clearInterval(webSocket.statsInterval);
+                }
+                webSocket.close();
+            }
+        }
+
+        // Event Listeners
+        document.getElementById('webrtc-button').addEventListener('click', () => {
+            const button = document.getElementById('webrtc-button');
+            if (button.textContent === 'Connect WebRTC') {
+                setupWebRTC();
+            } else {
+                webrtc_stop();
+                button.textContent = 'Connect WebRTC';
+            }
+        });
+        const ws_start_button = document.getElementById('ws-button')
+        ws_start_button.addEventListener('click', () => {
+            if (ws_start_button.textContent === 'Connect WebSocket') {
+                setupWebSocket();
+                ws_start_button.textContent = 'Stop';
+            } else {
+                ws_stop();
+                ws_start_button.textContent = 'Connect WebSocket';
+            }
+        });
+        document.addEventListener("beforeunload", () => {
+            ws_stop();
+            webrtc_stop();
+        });
+    </script>
+</body>
+
+</html>

requirements.txt

@@ -0,0 +1,6 @@
+fastrtc
+elevenlabs
+groq
+anthropic
+twilio
+python-dotenv