Upload 2 files

EchoAudio.py

@@ -0,0 +1,125 @@
+import asyncio
+import websockets
+from collections import deque
+import time
+import json
+import base64
+import uuid
+import numpy as np
+import sys
+
+# --- Configuration ---
+RECEIVE_HOST = "localhost"
+RECEIVE_PORT = 8765
+
+SEND_HOST = "localhost"
+SEND_PORT = 8766
+
+# The target delay in seconds.
+TARGET_DELAY_SECONDS = 2
+# The client sends a chunk of data every 1 second (1000ms).
+CHUNK_SEND_INTERVAL = 1.0
+
+# --- Server Logic ---
+async def receive_from_client_handler(websocket, internal_audio_queue):
+    """
+    Handles a single WebSocket connection for receiving audio from the client.
+    This server's only job is to receive and process data, then put it in a queue.
+    """
+    print(f"[{time.strftime('%H:%M:%S')}] Receive server: New client connected from {websocket.remote_address}")
+    try:
+        async for message in websocket:
+            print(f"[{time.strftime('%H:%M:%S')}] Receive server: Message received")
+            # The client sends a JSON string, so we need to parse it.
+            message_object = json.loads(message)
+            
+            # Decode the base64 audio data back to a binary array
+            audio_data_base64 = message_object['audioData']
+            audio_bytes = base64.b64decode(audio_data_base64)
+            
+            # Truncate to even length for int16 processing
+            audio_bytes = audio_bytes[:(len(audio_bytes) // 2) * 2]
+            
+            # Calculate loudness (RMS) of the audio chunk
+            audio_samples = np.frombuffer(audio_bytes, dtype=np.int16)
+            rms = np.sqrt(np.mean(np.square(audio_samples.astype(np.float64)))) if audio_samples.size > 0 else 0
+            rms = float(rms)
+            
+            # Add loudness and a new ID to the message object to send back
+            message_object['id'] = str(uuid.uuid4())
+            message_object['loudness'] = rms
+
+            await internal_audio_queue.put(message_object)
+            print(f"[{time.strftime('%H:%M:%S')}] Receive server: Received chunk #{message_object['chunkNumber']} with ID {message_object['id']}. Queue size: {internal_audio_queue.qsize()}")
+    except websockets.exceptions.ConnectionClosed as e:
+        print(f"[{time.strftime('%H:%M:%S')}] Receive server: Connection closed with code {e.code}")
+    except asyncio.CancelledError:
+        print(f"[{time.strftime('%H:%M:%S')}] Receive server: Task cancelled.")
+    except Exception as e:
+        print(f"[{time.strftime('%H:%M:%S')}] Receive server: Error: {e}")
+    finally:
+        print(f"[{time.strftime('%H:%M:%S')}] Receive server: Client disconnected.")
+
+async def send_to_client_handler(websocket, internal_audio_queue):
+    """
+    Handles a single WebSocket connection for sending delayed audio to the client.
+    This server's only job is to get data from the internal queue and send it.
+    """
+    print(f"[{time.strftime('%H:%M:%S')}] Send server: New client connected from {websocket.remote_address}")
+    internal_audio_buffer = deque()
+
+    try:
+        print(f"[{time.strftime('%H:%M:%S')}] Send server: Waiting for buffer to fill...")
+        # Fill the initial buffer. This blocks until enough chunks are available.
+        while len(internal_audio_buffer) < (TARGET_DELAY_SECONDS / CHUNK_SEND_INTERVAL):
+            chunk = await internal_audio_queue.get()
+            internal_audio_buffer.append(chunk)
+            print(f"[{time.strftime('%H:%M:%S')}] Send server: Filling buffer... Current size: {len(internal_audio_buffer)}")
+        
+        print(f"[{time.strftime('%H:%M:%S')}] Send server: Buffer filled. Starting to echo audio back.")
+
+        # Main loop to continuously process and send data
+        while True:
+            # Get the next chunk from the queue. This will block until a new chunk is available.
+            new_chunk = await internal_audio_queue.get()
+            
+            # Put the new chunk into the buffer and pop the oldest one
+            internal_audio_buffer.append(new_chunk)
+            chunk_to_send = internal_audio_buffer.popleft()
+            
+            await websocket.send(json.dumps(chunk_to_send))
+            print(f"[{time.strftime('%H:%M:%S')}] Send server: Sent chunk #{chunk_to_send['chunkNumber']} with ID {chunk_to_send['id']}. Buffer size: {len(internal_audio_buffer)}")
+
+    except websockets.exceptions.ConnectionClosed as e:
+        print(f"[{time.strftime('%H:%M:%S')}] Send server: Connection closed with code {e.code}")
+    except asyncio.CancelledError:
+        print(f"[{time.strftime('%H:%M:%S')}] Send server: Task cancelled.")
+    finally:
+        print(f"[{time.strftime('%H:%M:%S')}] Send server: Client disconnected.")
+
+# --- Main Server Startup ---
+async def main():
+    """
+    Starts both WebSocket servers concurrently and keeps them running.
+    """
+    print(f"[{time.strftime('%H:%M:%S')}] Starting servers...")
+    
+    # Create the internal queue for communication between servers
+    internal_audio_queue = asyncio.Queue()
+    
+    receive_server = await websockets.serve(lambda ws: receive_from_client_handler(ws, internal_audio_queue), RECEIVE_HOST, RECEIVE_PORT)
+    print(f"[{time.strftime('%H:%M:%S')}] Receive server started on ws://{RECEIVE_HOST}:{RECEIVE_PORT}")
+    
+    send_server = await websockets.serve(lambda ws: send_to_client_handler(ws, internal_audio_queue), SEND_HOST, SEND_PORT)
+    print(f"[{time.strftime('%H:%M:%S')}] Send server started on ws://{SEND_HOST}:{SEND_PORT}")
+
+    await asyncio.gather(receive_server.wait_closed(), send_server.wait_closed())
+
+if __name__ == "__main__":
+    if "win" in sys.platform:
+        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\nServer shutting down gracefully...")

client.html

@@ -0,0 +1,388 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Real-time Audio Stream</title>
+    <!-- Tailwind CSS for styling -->
+    <script src="https://cdn.tailwindcss.com"></script>
+    <style>
+        body {
+            font-family: "Inter", sans-serif;
+            background-color: #f3f4f6;
+        }
+    </style>
+</head>
+<body class="flex items-center justify-center min-h-screen">
+    <div class="bg-white p-8 rounded-xl shadow-lg w-full max-w-2xl mx-4">
+        <h1 class="text-3xl font-bold text-center mb-4 text-gray-800">Real-time Audio Stream</h1>
+        <p class="text-center text-gray-600 mb-6">
+            Capturing your microphone audio and sending it to a Python server. The server will echo the audio back with a 2-second delay.
+        </p>
+        
+        <!-- Status Indicator -->
+        <div id="status-container" class="flex items-center justify-center mb-6">
+            <span id="status-dot-send" class="block h-3 w-3 rounded-full mr-2"></span>
+            <span id="status-text-send" class="text-sm font-medium text-gray-700 mr-4">Connecting Send...</span>
+            <span id="status-dot-receive" class="block h-3 w-3 rounded-full mr-2"></span>
+            <span id="status-text-receive" class="text-sm font-medium text-gray-700">Connecting Receive...</span>
+        </div>
+
+        <!-- Control Buttons -->
+        <div class="flex justify-center space-x-4">
+            <button id="startButton" disabled class="bg-emerald-500 text-white font-semibold py-3 px-6 rounded-lg shadow-md hover:bg-emerald-600 transition-colors duration-200 focus:outline-none focus:ring-2 focus:ring-emerald-500 focus:ring-offset-2 disabled:bg-gray-400 disabled:cursor-not-allowed">
+                Start
+            </button>
+            <button id="stopButton" disabled class="bg-red-500 text-white font-semibold py-3 px-6 rounded-lg shadow-md hover:bg-red-600 transition-colors duration-200 focus:outline-none focus:ring-2 focus:ring-red-500 focus:ring-offset-2 disabled:bg-gray-400 disabled:cursor-not-allowed">
+                Stop
+            </button>
+        </div>
+
+        <!-- Debugging Log -->
+        <div class="mt-8">
+            <h2 class="text-xl font-semibold mb-2 text-gray-800">Chunk Information</h2>
+            <div id="chunk-log" class="bg-gray-100 p-4 rounded-lg overflow-y-scroll h-48 text-sm text-gray-700">
+                <p id="initial-log-message">Waiting for audio chunks...</p>
+            </div>
+            <p id="chunk-count" class="text-right text-xs text-gray-500 mt-2">Chunks received: 0</p>
+        </div>
+
+        <!-- Message box for user feedback -->
+        <div id="message-box" class="mt-6 text-sm text-center text-gray-500"></div>
+    </div>
+
+    <script>
+        // --- Configuration ---
+        const SEND_WEBSOCKET_URL = "ws://localhost:8765";
+        const RECEIVE_WEBSOCKET_URL = "ws://localhost:8766";
+        const CHUNK_DURATION = 1; // seconds per chunk
+
+        // --- DOM Elements ---
+        const startButton = document.getElementById('startButton');
+        const stopButton = document.getElementById('stopButton');
+        const statusDotSend = document.getElementById('status-dot-send');
+        const statusTextSend = document.getElementById('status-text-send');
+        const statusDotReceive = document.getElementById('status-dot-receive');
+        const statusTextReceive = document.getElementById('status-text-receive');
+        const messageBox = document.getElementById('message-box');
+        const chunkLog = document.getElementById('chunk-log');
+        const initialLogMessage = document.getElementById('initial-log-message');
+        const chunkCountDisplay = document.getElementById('chunk-count');
+
+        // --- Global State Variables ---
+        let audioContext;
+        let micSource;
+        let scriptNode;
+        let gainNode;
+        let accumulatedAudio = [];
+        let chunkSamples;
+        let sendWebSocket;
+        let receiveWebSocket;
+        let isRecording = false;
+        let audioQueue = [];
+        let isPlaying = false;
+        let nextPlayTime = 0;
+        let sendChunkCounter = 0;
+        let receiveChunkCounter = 0;
+        let expectedChunkNumber = 1;
+        
+        // --- Helper Functions ---
+        function setStatus(connection, dotColor, text) {
+            if (connection === 'send') {
+                statusDotSend.className = `block h-3 w-3 rounded-full mr-2 bg-${dotColor}-500`;
+                statusTextSend.textContent = text;
+            } else if (connection === 'receive') {
+                statusDotReceive.className = `block h-3 w-3 rounded-full mr-2 bg-${dotColor}-500`;
+                statusTextReceive.textContent = text;
+            }
+        }
+
+        function showMessage(text, isError = false) {
+            messageBox.textContent = text;
+            messageBox.className = `mt-6 text-sm text-center ${isError ? 'text-red-500' : 'text-gray-500'}`;
+        }
+        
+        function getCurrentTime() {
+            const now = new Date();
+            const hours = String(now.getHours()).padStart(2, '0');
+            const minutes = String(now.getMinutes()).padStart(2, '0');
+            const seconds = String(now.getSeconds()).padStart(2, '0');
+            return `${hours}:${minutes}:${seconds}`;
+        }
+
+        function appendChunkLog(chunkNumber, id, loudness, isSent = false, timeToReceive = null, timeToPlay = null) {
+            if (initialLogMessage) {
+                initialLogMessage.remove();
+            }
+            const logEntry = document.createElement('p');
+            if (isSent) {
+                logEntry.textContent = `[${getCurrentTime()}] SENT chunk #${chunkNumber}`;
+                logEntry.className = 'py-1 border-b border-gray-200 last:border-b-0 text-gray-500 italic';
+            } else {
+                let timeString = '';
+                if (timeToReceive !== null) {
+                    timeString += ` | Receive time: ${timeToReceive.toFixed(2)}ms`;
+                }
+                if (timeToPlay !== null) {
+                    timeString += ` | Play time: ${timeToPlay.toFixed(2)}ms`;
+                }
+                logEntry.textContent = `[${getCurrentTime()}] RECEIVED chunk #${chunkNumber}: Loudness: ${loudness.toFixed(2)}${timeString}`;
+                logEntry.className = 'py-1 border-b border-gray-200 last:border-b-0 font-bold text-gray-800';
+                
+                receiveChunkCounter++;
+                chunkCountDisplay.textContent = `Chunks received: ${receiveChunkCounter}`;
+            }
+            chunkLog.appendChild(logEntry);
+            chunkLog.scrollTop = chunkLog.scrollHeight;
+        }
+        
+        // --- Web Audio Playback Logic ---
+        function playbackLoop() {
+            if (!isPlaying) return;
+            
+            console.log('PlaybackLoop called, audioQueue length:', audioQueue.length, 'expected:', expectedChunkNumber);
+            
+            // Sort queue by chunkNumber
+            audioQueue.sort((a, b) => a.chunkNumber - b.chunkNumber);
+            
+            // Skip missing chunks
+            while (audioQueue.length > 0 && audioQueue[0].chunkNumber < expectedChunkNumber) {
+                audioQueue.shift();
+            }
+            
+            let played = false;
+            while (audioQueue.length > 0 && audioQueue[0].chunkNumber === expectedChunkNumber) {
+                console.log('Processing chunk #', audioQueue[0].chunkNumber);
+                played = true;
+                const chunkData = audioQueue.shift();
+                const { audioData, chunkNumber, loudness, receivedTime, id } = chunkData;
+
+                const playbackStartTime = performance.now();
+
+                try {
+                    const binaryString = window.atob(audioData);
+                    const len = binaryString.length;
+                    const bytes = new Uint8Array(len);
+                    for (let i = 0; i < len; i++) {
+                        bytes[i] = binaryString.charCodeAt(i);
+                    }
+                    const int16Arr = new Int16Array(bytes.buffer);
+                    const float32Arr = new Float32Array(int16Arr.length);
+                    for (let i = 0; i < int16Arr.length; i++) {
+                        float32Arr[i] = int16Arr[i] / 32768;
+                    }
+                    const audioBuffer = audioContext.createBuffer(
+                        1,
+                        float32Arr.length,
+                        audioContext.sampleRate
+                    );
+                    audioBuffer.copyToChannel(float32Arr, 0);
+
+                    const playbackEndTime = performance.now();
+                    const timeToPlay = playbackEndTime - playbackStartTime;
+                    
+                    const timeToReceive = receivedTime - chunkData.workerPostTime;
+                    appendChunkLog(chunkNumber, id, loudness, false, timeToReceive, timeToPlay);
+
+                    const source = audioContext.createBufferSource();
+                    source.buffer = audioBuffer;
+                    source.connect(audioContext.destination);
+                    
+                    const currentTime = audioContext.currentTime;
+                    const delay = nextPlayTime - currentTime;
+                    source.start(currentTime + Math.max(0, delay));
+                    
+                    nextPlayTime = Math.max(nextPlayTime, currentTime) + audioBuffer.duration;
+                    expectedChunkNumber++;
+                    
+                    // Set onended only if no more to play immediately
+                    if (audioQueue.length === 0 || audioQueue[0].chunkNumber !== expectedChunkNumber) {
+                        source.onended = playbackLoop;
+                    }
+                } catch (e) {
+                    console.error(`Error processing audio data for chunk #${chunkNumber}:`, e);
+                }
+            }
+            
+            if (!played) {
+                setTimeout(playbackLoop, 10);
+            }
+        }
+        
+        // --- Main WebSocket and Audio Logic ---
+        function connectWebSockets() {
+            startButton.disabled = true;
+            stopButton.disabled = true;
+            setStatus('send', 'yellow', 'Connecting Send...');
+            setStatus('receive', 'yellow', 'Connecting Receive...');
+            showMessage('Attempting to connect to both servers...');
+
+            try {
+                // Connect the send websocket
+                sendWebSocket = new WebSocket(SEND_WEBSOCKET_URL);
+                sendWebSocket.onopen = () => {
+                    console.log("Send WebSocket connection established.");
+                    setStatus('send', 'green', 'Send Connected');
+                    if (receiveWebSocket && receiveWebSocket.readyState === WebSocket.OPEN) {
+                        startButton.disabled = false;
+                        stopButton.disabled = true;
+                        showMessage('Successfully connected to both servers. You can now start recording.');
+                    }
+                };
+                sendWebSocket.onclose = () => {
+                    console.log("Send WebSocket connection closed.");
+                    setStatus('send', 'red', 'Send Disconnected');
+                    startButton.disabled = true;
+                    stopButton.disabled = true;
+                    isRecording = false;
+                };
+                sendWebSocket.onerror = (error) => {
+                    console.error("Send WebSocket error:", error);
+                    setStatus('send', 'red', 'Send Error');
+                    showMessage('Could not connect to the send server. Is the Python server running?', true);
+                };
+
+                // Connect the receive websocket
+                receiveWebSocket = new WebSocket(RECEIVE_WEBSOCKET_URL);
+                receiveWebSocket.onopen = () => {
+                    console.log("Receive WebSocket connection established.");
+                    setStatus('receive', 'green', 'Receive Connected');
+                    if (sendWebSocket && sendWebSocket.readyState === WebSocket.OPEN) {
+                        startButton.disabled = false;
+                        stopButton.disabled = true;
+                        showMessage('Successfully connected to both servers. You can now start recording.');
+                    }
+                };
+                receiveWebSocket.onmessage = (event) => {
+                    console.log('Receive WebSocket message received');
+                    const workerPostTime = performance.now();
+                    const chunkData = JSON.parse(event.data);
+                    const receivedTime = performance.now();
+                    chunkData.workerPostTime = workerPostTime;
+                    chunkData.receivedTime = receivedTime;
+                    audioQueue.push(chunkData);
+                    playbackLoop();
+                };
+                receiveWebSocket.onclose = () => {
+                    console.log("Receive WebSocket connection closed.");
+                    setStatus('receive', 'red', 'Receive Disconnected');
+                    startButton.disabled = true;
+                    stopButton.disabled = true;
+                    isRecording = false;
+                    isPlaying = false;
+                    audioQueue = [];
+                };
+                receiveWebSocket.onerror = (error) => {
+                    console.error("Receive WebSocket error.");
+                    setStatus('receive', 'red', 'Receive Error');
+                    showMessage('Could not connect to the receive server. Is the Python server running?', true);
+                };
+
+            } catch (e) {
+                console.error("Failed to create WebSockets:", e);
+                setStatus('send', 'red', 'Send Failed');
+                setStatus('receive', 'red', 'Receive Failed');
+                showMessage('An error occurred. Check your server URLs.', true);
+                startButton.disabled = true;
+                stopButton.disabled = true;
+            }
+        }
+
+        async function startRecording() {
+            if (sendWebSocket && sendWebSocket.readyState === WebSocket.OPEN) {
+                try {
+                    await audioContext.resume();
+                    
+                    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+                    micSource = audioContext.createMediaStreamSource(stream);
+                    scriptNode = audioContext.createScriptProcessor(4096, 1, 1);
+                    gainNode = audioContext.createGain();
+                    gainNode.gain.value = 0;
+
+                    micSource.connect(scriptNode);
+                    scriptNode.connect(gainNode);
+                    gainNode.connect(audioContext.destination);
+
+                    chunkSamples = Math.round(audioContext.sampleRate * CHUNK_DURATION);
+                    accumulatedAudio = [];
+                    scriptNode.onaudioprocess = (e) => {
+                        if (!isRecording) return;
+                        const data = e.inputBuffer.getChannelData(0);
+                        for (let i = 0; i < data.length; i++) {
+                            accumulatedAudio.push(data[i]);
+                        }
+                        while (accumulatedAudio.length >= chunkSamples) {
+                            const chunk = accumulatedAudio.splice(0, chunkSamples);
+                            const int16Arr = new Int16Array(chunk.length);
+                            for (let i = 0; i < chunk.length; i++) {
+                                int16Arr[i] = Math.max(-32768, Math.min(32767, chunk[i] * 32767));
+                            }
+                            const bytes = new Uint8Array(int16Arr.buffer);
+                            let binary = '';
+                            for (let i = 0; i < bytes.byteLength; i++) {
+                                binary += String.fromCharCode(bytes[i]);
+                            }
+                            const base64Data = btoa(binary);
+
+                            sendChunkCounter++;
+                            appendChunkLog(sendChunkCounter, null, null, true);
+
+                            const chunkToSend = {
+                                chunkNumber: sendChunkCounter,
+                                audioData: base64Data
+                            };
+                            if (sendWebSocket.readyState === WebSocket.OPEN) {
+                                sendWebSocket.send(JSON.stringify(chunkToSend));
+                            }
+                        }
+                    };
+
+                    console.log("Recording started.");
+                    startButton.disabled = true;
+                    stopButton.disabled = false;
+                    isRecording = true;
+                    showMessage("Recording... Please speak into your microphone.");
+                    
+                    // Start the playback loop
+                    if (!isPlaying) {
+                        isPlaying = true;
+                        nextPlayTime = audioContext.currentTime;
+                        playbackLoop();
+                    }
+
+                } catch (err) {
+                    console.error("Failed to start recording:", err);
+                    showMessage('Failed to start recording. Check console for errors.', true);
+                    startButton.disabled = false;
+                }
+            } else {
+                showMessage('Not connected to the send server. Please wait or refresh.', true);
+            }
+        }
+
+        function stopRecording() {
+            if (isRecording) {
+                isRecording = false;
+                isPlaying = false;
+                if (micSource) micSource.disconnect();
+                if (scriptNode) scriptNode.disconnect();
+                if (gainNode) gainNode.disconnect();
+                const tracks = micSource?.mediaStream.getTracks();
+                tracks?.forEach(track => track.stop());
+                accumulatedAudio = [];
+                console.log("Recording stopped.");
+                startButton.disabled = false;
+                stopButton.disabled = true;
+            }
+        }
+
+        startButton.addEventListener('click', startRecording);
+        stopButton.addEventListener('click', stopRecording);
+        
+        document.addEventListener('DOMContentLoaded', () => {
+            audioContext = new (window.AudioContext || window.webkitAudioContext)();
+            connectWebSockets();
+        });
+    </script>
+</body>
+</html>