feat: bidirectional audio

index.html

@@ -520,6 +520,22 @@
                             </svg>
                             <span id="muteText">Unmute</span>
                         </button>
+                        <button class="btn btn-mute muted" id="micBtn" onclick="toggleMic()" disabled>
+                            <svg id="micOffIcon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
+                                <line x1="1" y1="1" x2="23" y2="23"></line>
+                                <path d="M9 9v3a3 3 0 0 0 5.12 2.12M15 9.34V4a3 3 0 0 0-5.94-.6"></path>
+                                <path d="M17 16.95A7 7 0 0 1 5 12v-2m14 0v2c0 .76-.13 1.49-.35 2.17"></path>
+                                <line x1="12" y1="19" x2="12" y2="23"></line>
+                                <line x1="8" y1="23" x2="16" y2="23"></line>
+                            </svg>
+                            <svg id="micOnIcon" class="hidden" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
+                                <path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z"></path>
+                                <path d="M19 10v2a7 7 0 0 1-14 0v-2"></path>
+                                <line x1="12" y1="19" x2="12" y2="23"></line>
+                                <line x1="8" y1="23" x2="16" y2="23"></line>
+                            </svg>
+                            <span id="micText">Mic Off</span>
+                        </button>
                     </div>
                 </div>
             </div>
@@ -619,6 +635,10 @@
         // Audio mute state (for robot's audio playback)
         let isMuted = true; // Default to muted
 
+        // Microphone state (for speaking through the robot)
+        let localMicStream = null;
+        let isMicMuted = true; // Default mic off
+
         // Export functions
         window.loginToHuggingFace = loginToHuggingFace;
         window.logout = logout;
@@ -628,6 +648,7 @@
         window.playSound = playSound;
         window.playSoundPreset = playSoundPreset;
         window.toggleMute = toggleMute;
+        window.toggleMic = toggleMic;
 
         document.addEventListener('DOMContentLoaded', () => {
             initAuth();
@@ -814,10 +835,28 @@
             isMuted = true;
             updateMuteButton();
 
+            // Capture microphone for bidirectional audio (speak through robot)
+            try {
+                localMicStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+                localMicStream.getAudioTracks().forEach(t => t.enabled = false); // Start muted
+                isMicMuted = true;
+                updateMicButton();
+            } catch (e) {
+                console.warn('Microphone not available, speak-through-robot disabled:', e);
+                localMicStream = null;
+            }
+
             peerConnection = new RTCPeerConnection({
                 iceServers: [{ urls: 'stun:stun.l.google.com:19302' }]
             });
 
+            // Add mic track to PeerConnection (before SDP exchange)
+            if (localMicStream) {
+                for (const track of localMicStream.getAudioTracks()) {
+                    peerConnection.addTrack(track, localMicStream);
+                }
+            }
+
             peerConnection.ontrack = (e) => {
                 if (e.track.kind === 'video') {
                     const video = document.getElementById('remoteVideo');
@@ -868,8 +907,13 @@
             if (!peerConnection) return;
             try {
                 if (msg.sdp) {
-                    await peerConnection.setRemoteDescription(new RTCSessionDescription(msg.sdp));
-                    if (msg.sdp.type === 'offer') {
+                    let sdp = msg.sdp;
+                    // If we have a mic, ensure audio is sendrecv for bidirectional audio
+                    if (sdp.type === 'offer' && localMicStream) {
+                        sdp = { ...sdp, sdp: makeAudioBidirectional(sdp.sdp) };
+                    }
+                    await peerConnection.setRemoteDescription(new RTCSessionDescription(sdp));
+                    if (sdp.type === 'offer') {
                         const answer = await peerConnection.createAnswer();
                         await peerConnection.setLocalDescription(answer);
                         await sendToServer({ type: 'peer', sessionId: currentSessionId, sdp: { type: 'answer', sdp: answer.sdp } });
@@ -913,6 +957,13 @@
                 await sendToServer({ type: 'endSession', sessionId: currentSessionId });
             }
 
+            // Stop microphone
+            if (localMicStream) {
+                localMicStream.getTracks().forEach(t => t.stop());
+                localMicStream = null;
+            }
+            isMicMuted = true;
+
             if (peerConnection) peerConnection.close();
             if (dataChannel) dataChannel.close();
 
@@ -930,6 +981,7 @@
         function enableControls(enabled) {
             document.getElementById('btnPlaySound').disabled = !enabled;
             document.getElementById('muteBtn').disabled = !enabled;
+            document.getElementById('micBtn').disabled = !enabled || !localMicStream;
         }
 
         function toggleMute() {
@@ -957,6 +1009,45 @@
             }
         }
 
+        function toggleMic() {
+            if (!localMicStream) return;
+            isMicMuted = !isMicMuted;
+            localMicStream.getAudioTracks().forEach(t => t.enabled = !isMicMuted);
+            updateMicButton();
+        }
+
+        function updateMicButton() {
+            const btn = document.getElementById('micBtn');
+            const micOffIcon = document.getElementById('micOffIcon');
+            const micOnIcon = document.getElementById('micOnIcon');
+            const micText = document.getElementById('micText');
+
+            if (isMicMuted) {
+                btn.classList.add('muted');
+                micOffIcon.classList.remove('hidden');
+                micOnIcon.classList.add('hidden');
+                micText.textContent = 'Mic Off';
+            } else {
+                btn.classList.remove('muted');
+                micOffIcon.classList.add('hidden');
+                micOnIcon.classList.remove('hidden');
+                micText.textContent = 'Mic On';
+            }
+        }
+
+        function makeAudioBidirectional(sdp) {
+            // Change audio section direction from sendonly to sendrecv
+            // so the browser can send microphone audio to the robot
+            const lines = sdp.split('\r\n');
+            let inAudioSection = false;
+            return lines.map(line => {
+                if (line.startsWith('m=audio')) inAudioSection = true;
+                else if (line.startsWith('m=')) inAudioSection = false;
+                if (inAudioSection && line === 'a=sendonly') return 'a=sendrecv';
+                return line;
+            }).join('\r\n');
+        }
+
         // ===================== Robot State =====================
         function handleRobotMessage(data) {
             if (data.state) updateStateDisplay(data.state);