let pc; let ws; let localStream; let videoSender; let lastVideoBytes = 0; let lastVideoTs = 0; let currentMode = "vp8"; let autoLowCount = 0; let autoMidCount = 0; let autoHighCount = 0; const localVideo = document.getElementById("local"); const remoteVideo = document.getElementById("remote"); const statsBox = document.getElementById("stats"); const iceServers = [ { urls: "stun:stun.l.google.com:19302" } ]; document.getElementById("start").onclick = async () => { localStream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, video: { width: 640, height: 360, frameRate: 30 } }); localVideo.srcObject = localStream; statsBox.textContent = "Camera and microphone started."; }; document.getElementById("connect").onclick = async () => { if (!localStream) { await document.getElementById("start").onclick(); } pc = new RTCPeerConnection({ iceServers }); for (const track of localStream.getTracks()) { const sender = pc.addTrack(track, localStream); if (track.kind === "video") videoSender = sender; } pc.ontrack = (ev) => { remoteVideo.srcObject = ev.streams[0]; }; pc.onicecandidate = (ev) => { if (ev.candidate && ws?.readyState === WebSocket.OPEN) { ws.send(JSON.stringify({ type: "candidate", candidate: ev.candidate })); } }; const room = document.getElementById("room").value.trim() || "demo-room"; const proto = location.protocol === "https:" ? "wss" : "ws"; ws = new WebSocket(`${proto}://${location.host}/ws/${room}`); ws.onmessage = async (ev) => { const msg = JSON.parse(ev.data); if (msg.type === "joined") { statsBox.textContent = `Joined room. Peers in room: ${msg.peers}`; if (msg.peers >= 2) { await applyModePreferences(document.getElementById("mode").value, false); await makeOffer(); } return; } if (msg.type === "offer") { await pc.setRemoteDescription(msg.offer); await applyModePreferences(document.getElementById("mode").value, false); const answer = await pc.createAnswer(); await pc.setLocalDescription(answer); ws.send(JSON.stringify({ type: "answer", answer })); return; } if (msg.type === "answer") { await pc.setRemoteDescription(msg.answer); return; } if (msg.type === "candidate") { try { await pc.addIceCandidate(msg.candidate); } catch (_) {} } }; setInterval(logAndControlStats, 1000); }; document.getElementById("applyMode").onclick = async () => { await applyModePreferences(document.getElementById("mode").value, true); }; async function makeOffer() { if (!pc || !ws || ws.readyState !== WebSocket.OPEN) return; if (pc.signalingState !== "stable") return; const offer = await pc.createOffer(); await pc.setLocalDescription(offer); ws.send(JSON.stringify({ type: "offer", offer })); } async function applyModePreferences(mode, renegotiate = true) { if (!pc || !videoSender) return; currentMode = mode; if (mode === "vp8") { preferVideoCodec("video/VP8"); await setVideoParams({ maxBitrate: 1500000, maxFramerate: 30, scaleResolutionDownBy: 1 }); await replaceWithCameraTrack(); } if (mode === "av1") { preferVideoCodec("video/AV1"); await setVideoParams({ maxBitrate: 160000, maxFramerate: 10, scaleResolutionDownBy: 4 }); await replaceWithCameraTrack(); } if (mode === "ai") { await setVideoParams({ maxBitrate: 70000, maxFramerate: 15, scaleResolutionDownBy: 2 }); await replaceWithAiTrack(); } if (renegotiate) { await makeOffer(); } } function preferVideoCodec(mimeType) { const transceiver = pc.getTransceivers().find(t => t.sender === videoSender); if (!transceiver || !RTCRtpSender.getCapabilities) return; const caps = RTCRtpSender.getCapabilities("video"); if (!caps?.codecs?.length) return; const preferred = [ ...caps.codecs.filter(c => c.mimeType.toLowerCase() === mimeType.toLowerCase()), ...caps.codecs.filter(c => c.mimeType.toLowerCase() !== mimeType.toLowerCase()) ]; if (preferred.length) { transceiver.setCodecPreferences(preferred); } } async function setVideoParams({ maxBitrate, maxFramerate, scaleResolutionDownBy }) { try { const p = videoSender.getParameters(); p.encodings = p.encodings?.length ? p.encodings : [{}]; Object.assign(p.encodings[0], { maxBitrate, maxFramerate, scaleResolutionDownBy }); await videoSender.setParameters(p); } catch (e) { console.warn("Could not set sender params:", e); } } async function replaceWithCameraTrack() { const track = localStream.getVideoTracks()[0]; if (track) { await videoSender.replaceTrack(track); localVideo.srcObject = localStream; } } async function replaceWithAiTrack() { const audioTrack = localStream.getAudioTracks()[0]; if (!audioTrack) { throw new Error("No local audio track available."); } statsBox.textContent = "Mode 3: capturing face/audio and calling /api/thg..."; const faceBlob = await captureFaceFrame(localVideo); const audioBlob = await recordAudioChunk(audioTrack, 3000); const form = new FormData(); form.append("face", faceBlob, "face.png"); form.append("audio", audioBlob, "audio.webm"); const res = await fetch("/api/thg", { method: "POST", body: form }); if (!res.ok) { const err = await res.text(); throw new Error(err); } const mp4 = await res.blob(); const aiVideo = document.createElement("video"); aiVideo.src = URL.createObjectURL(mp4); aiVideo.loop = true; aiVideo.muted = true; aiVideo.playsInline = true; await aiVideo.play(); const aiStream = aiVideo.captureStream ? aiVideo.captureStream(15) : aiVideo.mozCaptureStream(15); const aiTrack = aiStream.getVideoTracks()[0]; if (!aiTrack) { throw new Error("Could not capture AI video stream."); } await videoSender.replaceTrack(aiTrack); localVideo.srcObject = aiStream; statsBox.textContent = "Mode 3 active: AI video track replaced."; } function captureFaceFrame(video) { return new Promise((resolve, reject) => { const canvas = document.createElement("canvas"); canvas.width = video.videoWidth || 640; canvas.height = video.videoHeight || 360; const ctx = canvas.getContext("2d"); ctx.drawImage(video, 0, 0, canvas.width, canvas.height); canvas.toBlob((blob) => { if (!blob) reject(new Error("Could not capture face frame.")); else resolve(blob); }, "image/png"); }); } function recordAudioChunk(track, ms) { return new Promise((resolve, reject) => { const stream = new MediaStream([track]); const mimeType = MediaRecorder.isTypeSupported("audio/webm;codecs=opus") ? "audio/webm;codecs=opus" : ""; const rec = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); const chunks = []; rec.ondataavailable = (e) => { if (e.data.size > 0) chunks.push(e.data); }; rec.onerror = (e) => reject(e.error || e); rec.onstop = () => { resolve(new Blob(chunks, { type: "audio/webm" })); }; rec.start(); setTimeout(() => rec.stop(), ms); }); } async function logAndControlStats() { if (!pc) return; const report = await pc.getStats(); let videoKbps = 0; let availableKbps = null; let frames = 0; let codecId = ""; report.forEach(s => { if (s.type === "outbound-rtp" && s.kind === "video") { const now = s.timestamp; const bytes = s.bytesSent || 0; if (lastVideoTs && now > lastVideoTs) { videoKbps = 8 * (bytes - lastVideoBytes) / (now - lastVideoTs); } lastVideoBytes = bytes; lastVideoTs = now; frames = s.framesEncoded || 0; codecId = s.codecId || ""; } if ( s.type === "candidate-pair" && s.state === "succeeded" && (s.nominated || s.selected) ) { if (s.availableOutgoingBitrate) { availableKbps = s.availableOutgoingBitrate / 1000; } } }); if (document.getElementById("autoMode").checked && availableKbps) { await maybeAutoSwitch(availableKbps); } statsBox.textContent = `activeMode=${currentMode}\n` + `videoSend=${videoKbps.toFixed(1)} kbps\n` + `availableOutgoing=${availableKbps ? availableKbps.toFixed(1) + " kbps" : "not exposed"}\n` + `framesEncoded=${frames}\n` + `codecId=${codecId}\n` + `pcState=${pc.connectionState}`; } async function maybeAutoSwitch(availableKbps) { if (availableKbps < 90) { autoLowCount++; autoMidCount = 0; autoHighCount = 0; } else if (availableKbps < 400) { autoMidCount++; autoLowCount = 0; autoHighCount = 0; } else { autoHighCount++; autoLowCount = 0; autoMidCount = 0; } if (autoLowCount >= 3 && currentMode !== "ai") { document.getElementById("mode").value = "ai"; await applyModePreferences("ai", true); } if (autoMidCount >= 3 && currentMode !== "av1") { document.getElementById("mode").value = "av1"; await applyModePreferences("av1", true); } if (autoHighCount >= 3 && currentMode !== "vp8") { document.getElementById("mode").value = "vp8"; await applyModePreferences("vp8", true); } }