VibeVoice / static /app.js
RAM2106's picture
Initial VibeVoice Studio commit
cd5198d
Raw
History Blame Contribute Delete
28.1 kB
/**
* VibeVoice Studio - Front-End Audio Engine & UI Controller
* Handles WebSocket PCM streaming, gapless scheduling, Web Audio API,
* microphone recording, canvas visualization, and dashboard state management.
*/
// Global App State
let audioCtx = null;
let analyserNode = null;
let gainNode = null;
let ws = null;
let currentAudioSource = null;
// Streaming Audio Queue Variables (Gapless Playback)
let playbackQueue = [];
let nextPlayTime = 0;
let isStreamingActive = false;
let scheduledBufferSources = [];
const SAMPLING_RATE = 24000; // VibeVoice standard samplerate
// Voice Recording State (Voice Cloning)
let mediaRecorders = [null, null];
let audioChunks = [[], []];
let voicePaths = [null, null];
// HTML Elements Selection
const modelStatusVal = document.getElementById("modelStatusVal");
const modelDot = document.getElementById("modelDot");
const btnToggleModel = document.getElementById("btnToggleModel");
const connBadge = document.getElementById("connBadge");
const connIcon = document.getElementById("connIcon");
const connText = document.getElementById("connText");
const txtScript = document.getElementById("txtScript");
const chkStreamMode = document.getElementById("chkStreamMode");
const btnSynthesize = document.getElementById("btnSynthesize");
const btnStop = document.getElementById("btnStop");
const agentOrb = document.getElementById("agentOrb");
const agentStateText = document.getElementById("agentStateText");
const agentStateSub = document.getElementById("agentStateSub");
const canvasWaveform = document.getElementById("canvasWaveform");
const canvasCtx = canvasWaveform.getContext("2d");
// Playback Toolbar Elements
const audioToolbar = document.getElementById("audioToolbar");
const btnToolbarPlay = document.getElementById("btnToolbarPlay");
const sliderTimeline = document.getElementById("sliderTimeline");
const progressTimelineFill = document.getElementById("progressTimelineFill");
const lblTimeStart = document.getElementById("lblTimeStart");
const lblTimeEnd = document.getElementById("lblTimeEnd");
const sliderVolume = document.getElementById("sliderVolume");
const iconVolume = document.getElementById("iconVolume");
const selectSpeed = document.getElementById("selectSpeed");
// Voice Cloning Slots Elements
const spkName0 = document.getElementById("spkName0");
const btnRecord0 = document.getElementById("btnRecord0");
const fileVoice0 = document.getElementById("fileVoice0");
const clonedStatus0 = document.getElementById("clonedStatus0");
const audioPreview0 = document.getElementById("audioPreview0");
const spkName1 = document.getElementById("spkName1");
const btnRecord1 = document.getElementById("btnRecord1");
const fileVoice1 = document.getElementById("fileVoice1");
const clonedStatus1 = document.getElementById("clonedStatus1");
const audioPreview1 = document.getElementById("audioPreview1");
const consoleBody = document.getElementById("consoleBody");
const btnClearLogs = document.getElementById("btnClearLogs");
// Presets Elements
const presetPodcast = document.getElementById("presetPodcast");
const presetAssistant = document.getElementById("presetAssistant");
const presetNews = document.getElementById("presetNews");
// ==========================================================================
// Initialization & Server Sync
// ==========================================================================
window.addEventListener("DOMContentLoaded", () => {
log("system", "Initializing Audio Engines...");
checkServerStatus();
setupCanvasVisualizer();
bindEvents();
});
// Logs messages into the built-in Console Log card on UI
function log(type, message) {
const timeStr = new Date().toLocaleTimeString();
const line = document.createElement("div");
line.className = `log-line ${type}`;
line.innerHTML = `[${timeStr}] ${message}`;
consoleBody.appendChild(line);
consoleBody.scrollTop = consoleBody.scrollHeight;
}
let statusInterval = null;
// Queries server backend to find if model weights are loaded
async function checkServerStatus() {
try {
const res = await fetch("/api/status");
const data = await res.json();
updateModelUI(data.use_real_model, data.loaded, data.loading, data.error);
// Auto poll if loading
if (data.loading) {
if (!statusInterval) {
statusInterval = setInterval(checkServerStatus, 3000);
}
} else {
if (statusInterval) {
clearInterval(statusInterval);
statusInterval = null;
}
}
// Try opening websocket streaming channel if not already connected/connecting
if (!ws || ws.readyState === WebSocket.CLOSED) {
initWebSocket();
}
} catch (e) {
log("error", "Failed to connect to VibeVoice backend API server.");
updateModelUI(false, false, false, "Offline");
if (statusInterval) {
clearInterval(statusInterval);
statusInterval = null;
}
}
}
function updateModelUI(useReal, loaded, loading, error) {
if (useReal) {
if (loading) {
modelStatusVal.innerText = "Downloading AI weights (1.2GB)...";
modelDot.className = "status-dot yellow animate-pulse";
btnToggleModel.classList.add("active");
log("info", "VibeVoice Model weights downloading from Hugging Face. Please stand by...");
} else if (loaded) {
modelStatusVal.innerText = "VibeVoice-0.5B AI";
modelDot.className = "status-dot green";
btnToggleModel.classList.add("active");
log("success", "VibeVoice Model weights loaded successfully. Zero-shot cloning activated!");
} else if (error) {
modelStatusVal.innerText = "Error Loading AI";
modelDot.className = "status-dot red";
btnToggleModel.classList.remove("active");
log("error", `Model Load Error: ${error}`);
}
} else {
modelStatusVal.innerText = "Demo Synthesizer";
modelDot.className = "status-dot green";
btnToggleModel.classList.remove("active");
}
}
// Handles switching model modes on clicking the Toggle button
btnToggleModel.addEventListener("click", async () => {
const isActive = btnToggleModel.classList.contains("active");
const enable = !isActive;
log("info", `Requesting server mode toggle: ${enable ? "Enable AI Model" : "Enable Demo Synthesizer"}`);
try {
btnToggleModel.disabled = true;
const res = await fetch(`/api/toggle-model?enable=${enable}`, { method: "POST" });
const data = await res.json();
btnToggleModel.disabled = false;
checkServerStatus();
} catch (e) {
btnToggleModel.disabled = false;
log("error", "Error toggling AI Model mode.");
}
});
// Initialize real-time WebSockets
function initWebSocket() {
const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
const host = window.location.host;
const wsUrl = `${protocol}//${host}/api/stream`;
log("system", `Opening WebSocket stream canal: ${wsUrl}`);
ws = new WebSocket(wsUrl);
ws.onopen = () => {
connBadge.className = "connection-badge online";
connIcon.className = "fa-solid fa-link";
connText.innerText = "Connected";
log("success", "Real-time streaming audio WebSocket channel ESTABLISHED.");
};
ws.onclose = () => {
connBadge.className = "connection-badge";
connIcon.className = "fa-solid fa-link-slash";
connText.innerText = "Offline";
log("error", "Real-time streaming WebSocket channel disconnected.");
// Try reconnecting in 5 seconds
setTimeout(initWebSocket, 5000);
};
ws.onerror = (e) => {
log("error", "WebSocket channel encountered error. Resetting...");
};
ws.onmessage = async (event) => {
// Handle incoming WebSocket messages (could be text status or binary audio)
if (typeof event.data === "string") {
const data = JSON.parse(event.data);
if (data.type === "word") {
log("word", `Agent spoke word: <strong style="font-size:12px;">${data.word}</strong>`);
} else if (data.type === "done") {
log("success", "WebSocket audio stream finished downloading.");
} else if (data.type === "error") {
log("error", `Server Stream Error: ${data.message}`);
stopPlayback();
}
} else {
// Audio chunk arrived as binary raw data
const arrayBuffer = await event.data.arrayBuffer();
// Convert 16-bit signed PCM buffer to Float32 array
const float32Array = convertPCM16ToFloat32(new Int16Array(arrayBuffer));
if (isStreamingActive) {
playbackQueue.push(float32Array);
scheduleNextChunk();
}
}
};
}
// Helper to convert 16-bit integer bytes to floating values between -1.0 and 1.0
function convertPCM16ToFloat32(int16Array) {
const float32 = new Float32Array(int16Array.length);
for (let i = 0; i < int16Array.length; i++) {
// Divide by max positive value of 16-bit integer (32767)
float32[i] = int16Array[i] / 32767.0;
}
return float32;
}
// ==========================================================================
// Web Audio Context & Playback Engine (Pillar 2 & 3)
// ==========================================================================
function initAudioContext() {
if (audioCtx) return;
// Create modern AudioContext inside browser
audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: SAMPLING_RATE });
// Create Analyser Node for the Oscilloscope Waveform Canvas
analyserNode = audioCtx.createAnalyser();
analyserNode.fftSize = 256;
// Create Gain Node for volume adjustments
gainNode = audioCtx.createGain();
setVolume(sliderVolume.value / 100);
// Connect nodes together: Source -> Gain -> Analyser -> Output Speakers
gainNode.connect(analyserNode);
analyserNode.connect(audioCtx.destination);
log("system", `Browser AudioContext initiated at ${SAMPLING_RATE}Hz.`);
}
function setVolume(val) {
if (gainNode && audioCtx) {
gainNode.gain.setValueAtTime(val, audioCtx.currentTime);
// Toggle volume icon based on level
if (val === 0) iconVolume.className = "fa-solid fa-volume-mute";
else if (val < 0.5) iconVolume.className = "fa-solid fa-volume-low";
else iconVolume.className = "fa-solid fa-volume-high";
}
}
// Streaming Scheduling: Gapless stitched playing of successive floats
function startStreamingPlayback() {
initAudioContext();
if (audioCtx.state === "suspended") {
audioCtx.resume();
}
playbackQueue = [];
scheduledBufferSources = [];
nextPlayTime = audioCtx.currentTime + 0.15; // Brief offset padding buffer to start
isStreamingActive = true;
btnSynthesize.disabled = true;
btnStop.disabled = false;
setAgentOrbState("speaking", "STREAMING VOICE", "Synthesizing and playing real-time speech...");
log("info", "Playback stream started. Waiting for speech chunks...");
}
function scheduleNextChunk() {
if (!isStreamingActive || playbackQueue.length === 0) return;
const chunkFloats = playbackQueue.shift();
if (chunkFloats.length === 0) return;
// Create custom Audio Buffer for the float data
const buffer = audioCtx.createBuffer(1, chunkFloats.length, SAMPLING_RATE);
buffer.copyToChannel(chunkFloats, 0);
// Create Source Node
const sourceNode = audioCtx.createBufferSource();
sourceNode.buffer = buffer;
// Apply playback speed modifier selected by user
sourceNode.playbackRate.value = parseFloat(selectSpeed.value);
// Stitch chunk directly onto Gain node
sourceNode.connect(gainNode);
// Precise scheduling time calculation
const startTime = Math.max(audioCtx.currentTime, nextPlayTime);
sourceNode.start(startTime);
// Keep track of active playing nodes so we can cut off audio if requested
scheduledBufferSources.push(sourceNode);
// Update scheduler deadline
const chunkDuration = buffer.duration / sourceNode.playbackRate.value;
nextPlayTime = startTime + chunkDuration;
// Release reference on completion
sourceNode.onended = () => {
const idx = scheduledBufferSources.indexOf(sourceNode);
if (idx > -1) scheduledBufferSources.splice(idx, 1);
// If queue is exhausted and no active audio is left, reset agent to idle
if (scheduledBufferSources.length === 0 && playbackQueue.length === 0 && isStreamingActive) {
stopPlayback();
log("info", "Agent finished speaking.");
}
};
}
// Stops all active sounds and resets the dashboard execution state
function stopPlayback() {
isStreamingActive = false;
playbackQueue = [];
// Halt all audio buffers immediately
scheduledBufferSources.forEach(source => {
try { source.stop(); } catch(e) {}
});
scheduledBufferSources = [];
// Stop standard player if active
if (currentAudioSource) {
try { currentAudioSource.stop(); } catch(e) {}
currentAudioSource = null;
}
btnSynthesize.disabled = false;
btnStop.disabled = true;
setAgentOrbState("idle", "AGENT READY", "Configure scripts and trigger generation below");
audioToolbar.classList.add("hidden");
log("system", "Agent audio playback halted.");
}
btnStop.addEventListener("click", stopPlayback);
// Sets the visual status and glowing styles of the central agent orb
function setAgentOrbState(stateClass, title, subtitle) {
agentOrb.className = `agent-orb ${stateClass}`;
agentStateText.innerText = title;
agentStateSub.innerText = subtitle;
}
// ==========================================================================
// Canvas Oscilloscope Visualizer Rendering (Pillar 3)
// ==========================================================================
function setupCanvasVisualizer() {
const width = canvasWaveform.width;
const height = canvasWaveform.height;
function draw() {
requestAnimationFrame(draw);
canvasCtx.fillStyle = "rgba(7, 9, 14, 0.25)"; // Semitransparent sweep back
canvasCtx.fillRect(0, 0, width, height);
let dataArray = null;
let bufferLength = 0;
if (analyserNode && isAudioPlaying()) {
bufferLength = analyserNode.frequencyBinCount;
dataArray = new Uint8Array(bufferLength);
analyserNode.getByteTimeDomainData(dataArray);
}
canvasCtx.lineWidth = 2.5;
// Apply beautiful purple-to-cyan gradient stroke to waveform curves
const gradient = canvasCtx.createLinearGradient(0, 0, width, 0);
gradient.addColorStop(0, "hsl(265, 88%, 64%)"); // Purple
gradient.addColorStop(0.5, "hsl(185, 95%, 48%)"); // Cyan
gradient.addColorStop(1, "hsl(330, 92%, 60%)"); // Pink
canvasCtx.strokeStyle = gradient;
canvasCtx.shadowBlur = isAudioPlaying() ? 8 : 0;
canvasCtx.shadowColor = "rgba(0, 242, 254, 0.4)";
canvasCtx.beginPath();
if (dataArray) {
// Draw actual audio ripples
const sliceWidth = width / bufferLength;
let x = 0;
for (let i = 0; i < bufferLength; i++) {
const v = dataArray[i] / 128.0;
const y = (v * height) / 2;
if (i === 0) {
canvasCtx.moveTo(x, y);
} else {
canvasCtx.lineTo(x, y);
}
x += sliceWidth;
}
} else {
// Draw a calm breathing idle baseline
const points = 40;
const sliceWidth = width / points;
let x = 0;
const time = Date.now() * 0.003;
for (let i = 0; i < points; i++) {
// Subtle sine rhythm for visual engagement when silent
const breathingFactor = Math.sin(time) * 0.2 + 0.8;
const y = height / 2 + Math.sin(i * 0.15 + time * 2) * (2.5 * breathingFactor);
if (i === 0) {
canvasCtx.moveTo(x, y);
} else {
canvasCtx.lineTo(x, y);
}
x += sliceWidth;
}
}
canvasCtx.lineTo(width, height / 2);
canvasCtx.stroke();
}
draw();
}
function isAudioPlaying() {
return isStreamingActive || currentAudioSource !== null;
}
// ==========================================================================
// Microphone Recorder & Speaker Voice Cloning (Pillar 4)
// ==========================================================================
async function setupRecording(speakerId) {
const btnRecord = speakerId === 0 ? btnRecord0 : btnRecord1;
const isRecording = btnRecord.classList.contains("recording");
if (isRecording) {
// Stop active recording
if (mediaRecorders[speakerId]) {
mediaRecorders[speakerId].stop();
btnRecord.classList.remove("recording");
btnRecord.innerHTML = `<i class="fa-solid fa-microphone"></i> Record Mic`;
setAgentOrbState("idle", "AGENT READY", "Voice profile recorded. Transmitting to backend...");
}
} else {
// Start recording
try {
log("info", `Requesting microphone access for Speaker ${speakerId}...`);
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
initAudioContext();
setAgentOrbState("listening", "LISTENING...", `Say a short 10-15s dialogue sentence to clone voice profile.`);
audioChunks[speakerId] = [];
const mediaRecorder = new MediaRecorder(stream);
mediaRecorders[speakerId] = mediaRecorder;
mediaRecorder.ondataavailable = (event) => {
audioChunks[speakerId].push(event.data);
};
mediaRecorder.onstop = async () => {
log("info", "Processing audio chunks...");
const audioBlob = new Blob(audioChunks[speakerId], { type: "audio/wav" });
// Release microphore stream tracks
stream.getTracks().forEach(track => track.stop());
// Upload recorded WAV profile to server
const spkName = speakerId === 0 ? spkName0.value : spkName1.value;
await uploadVoiceProfile(audioBlob, speakerId, spkName);
};
mediaRecorder.start();
btnRecord.classList.add("recording");
btnRecord.innerHTML = `<i class="fa-solid fa-square"></i> Stop Mic`;
log("success", "Microphone recording started. Speak clearly now!");
} catch (e) {
log("error", `Could not record voice: ${e.message}`);
setAgentOrbState("idle", "AGENT READY", "Microphone permission denied or source invalid.");
}
}
}
// Transmit Voice profile file to FastAPI backend
async function uploadVoiceProfile(blob, speakerId, name) {
const formData = new FormData();
formData.append("file", blob, `mic_speaker_${speakerId}.wav`);
formData.append("speaker_name", name);
try {
log("info", "Uploading voice clone prompt to server database...");
const res = await fetch("/api/upload-voice", {
method: "POST",
body: formData
});
const data = await res.json();
if (data.status === "success") {
voicePaths[speakerId] = data.voice_path;
// Show preview player
const previewPlayer = speakerId === 0 ? audioPreview0 : audioPreview1;
const statusBox = speakerId === 0 ? clonedStatus0 : clonedStatus1;
previewPlayer.src = `/static/cloned_voices/${data.filename}`;
statusBox.classList.remove("hidden");
log("success", `Voice cloned successfully under speaker code "${name}"!`);
}
} catch(e) {
log("error", "Error uploading voice cloning profile.");
}
}
// Wire File Selector uploads for pre-existing WAV voice samples
async function handleFileSelect(e, speakerId) {
const file = e.target.files[0];
if (!file) return;
log("info", `Uploading selected file: ${file.name}`);
const name = speakerId === 0 ? spkName0.value : spkName1.value;
await uploadVoiceProfile(file, speakerId, name);
}
// ==========================================================================
// Speech Synthesis Triggers (REST vs WebSocket Stream)
// ==========================================================================
async function triggerSynthesis() {
const text = txtScript.value.trim();
if (!text) {
log("error", "Cannot synthesize speech. Script text field is empty!");
return;
}
initAudioContext();
stopPlayback(); // Clean up existing audio cycles
const isStream = chkStreamMode.checked;
const speaker_voices = {};
if (voicePaths[0]) speaker_voices["0"] = voicePaths[0];
if (voicePaths[1]) speaker_voices["1"] = voicePaths[1];
if (isStream) {
// Mode A: Real-Time WebSockets Streaming
if (!ws || ws.readyState !== WebSocket.OPEN) {
log("error", "WebSocket channel is not active. Streaming offline.");
return;
}
startStreamingPlayback();
// Push script parameters to WebSocket
ws.send(JSON.stringify({
text: text,
voice_sample_path: voicePaths[0], // primary voice clone prompt path
speaker_id: 0,
speaker_voices: speaker_voices
}));
} else {
// Mode B: Standard Full WAV synthesis
try {
log("info", "Initiating complete script compilation... Please wait...");
setAgentOrbState("thinking", "COMPILING SCRIPT", "Executing neural modeling and saving audio...");
btnSynthesize.disabled = true;
const res = await fetch("/api/generate", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
text: text,
voice_sample_path: voicePaths[0],
speaker_id: 0,
speaker_voices: speaker_voices
})
});
const data = await res.json();
btnSynthesize.disabled = false;
if (data.status === "success") {
log("success", `Script generated successfully using engine mode: [${data.mode}]`);
loadTraditionalPlayer(data.audio_url);
} else {
throw new Error("API return code was not successful.");
}
} catch (e) {
btnSynthesize.disabled = false;
setAgentOrbState("idle", "AGENT READY", "Dialogue generation failed.");
log("error", `Synthesis generation failed: ${e.message}`);
}
}
}
// Controls standard player execution for downloaded WAV files
async function loadTraditionalPlayer(audioUrl) {
try {
log("info", "Loading generated voice track into audio player...");
const res = await fetch(audioUrl);
const arrayBuffer = await res.arrayBuffer();
audioCtx.decodeAudioData(arrayBuffer, (decodedBuffer) => {
currentAudioSource = audioCtx.createBufferSource();
currentAudioSource.buffer = decodedBuffer;
currentAudioSource.connect(gainNode);
currentAudioSource.start(0);
setAgentOrbState("speaking", "PLAYING AUDIO", "Playing compiled high-fidelity voice track...");
audioToolbar.classList.remove("hidden");
btnStop.disabled = false;
btnSynthesize.disabled = true;
// Set up player timeline meters
lblTimeStart.innerText = "00:00";
lblTimeEnd.innerText = formatTime(decodedBuffer.duration);
sliderTimeline.max = Math.floor(decodedBuffer.duration);
sliderTimeline.value = 0;
const startTime = audioCtx.currentTime;
const updateTimeline = () => {
if (currentAudioSource) {
const elapsed = audioCtx.currentTime - startTime;
sliderTimeline.value = Math.floor(elapsed);
progressTimelineFill.style.width = `${(elapsed / decodedBuffer.duration) * 100}%`;
lblTimeStart.innerText = formatTime(elapsed);
if (elapsed < decodedBuffer.duration) {
requestAnimationFrame(updateTimeline);
}
}
};
updateTimeline();
currentAudioSource.onended = () => {
stopPlayback();
};
}, (err) => {
log("error", "Error decoding audio buffer track.");
});
} catch(e) {
log("error", "Failed to compile compiled WAV player.");
}
}
function formatTime(seconds) {
const m = Math.floor(seconds / 60).toString().padStart(2, '0');
const s = Math.floor(seconds % 60).toString().padStart(2, '0');
return `${m}:${s}`;
}
// ==========================================================================
// Event Binding & Presets Loading (Dashboard HUD controls)
// ==========================================================================
function bindEvents() {
// Primary execution actions
btnSynthesize.addEventListener("click", triggerSynthesis);
// Voice profiling recording hooks
btnRecord0.addEventListener("click", () => setupRecording(0));
btnRecord1.addEventListener("click", () => setupRecording(1));
fileVoice0.addEventListener("change", (e) => handleFileSelect(e, 0));
fileVoice1.addEventListener("change", (e) => handleFileSelect(e, 1));
// Standard audio adjustment nodes
sliderVolume.addEventListener("input", (e) => {
setVolume(e.target.value / 100);
});
btnClearLogs.addEventListener("click", () => {
consoleBody.innerHTML = `<div class="log-line system">[System] Console buffer cleared.</div>`;
});
// Presets loaders
presetPodcast.addEventListener("click", () => {
txtScript.value = `Speaker 0: Welcome back to the Vibe Voice podcast segment. Today we are cloning human expressions.
Speaker 1: That is right! The 0.5B model manages this dialogue transition on a local CPU extremely fast.
Speaker 0: Incredible! It sounds like two real people sitting in this glassmorphic studio together.`;
log("info", "Loaded preset script: [Podcast Dialogue]");
});
presetAssistant.addEventListener("click", () => {
txtScript.value = `Speaker 0: Hello, I am your personalized voice assistant. How can I help you customize your Dine Direct orders today?`;
log("info", "Loaded preset script: [AI Voice Assistant]");
});
presetNews.addEventListener("click", () => {
txtScript.value = `Speaker 1: Breaking news from Microsoft Research. The Vibe Voice speech synthesis models are now live on local devices.`;
log("info", "Loaded preset script: [News Broadcast]");
});
}