import React, { useState, useEffect, useRef } from 'react'; import LandingPage from './LandingPage'; import { supabase } from './supabaseClient'; import { Mic, Upload, Play, Square, Volume2, VolumeX, Terminal, Settings, Radio, Info, AlertCircle, CheckCircle, Sparkles, Power, Loader2, Trash2, Podcast, Headset, Newspaper, Volume1, PlayCircle, Plus, Music, Edit2 } from 'lucide-react'; const SAMPLING_RATE = 24000; function App() { const [currentView, setCurrentView] = useState('landing'); const [savedVoices, setSavedVoices] = useState([]); const [podcastHistory, setPodcastHistory] = useState([]); // Model & Server Status States const [modelStatus, setModelStatus] = useState({ useRealModel: true, modelId: 'microsoft/VibeVoice-Realtime-0.5B', loaded: false, loading: true, error: null, device: 'CPU' }); const [wsConnected, setWsConnected] = useState(false); const [togglingModel, setTogglingModel] = useState(false); // Script & Synthesis States const [script, setScript] = useState( "Speaker 0: Welcome to Vibe Voice Studio. I am cloned from your custom voice sample.\nSpeaker 1: That is incredible! The transition between speakers sounds so smooth and natural.\nSpeaker 0: Yes, Microsoft's continuous acoustic tokenizers make multi-speaker podcasts sound beautiful." ); const [streamMode, setStreamMode] = useState(true); const [agentState, setAgentState] = useState({ status: 'idle', // idle, thinking, speaking, listening title: 'AGENT READY', sub: 'Configure scripts and trigger generation below' }); // Audio Playback UI States const [isAudioPlayingState, setIsAudioPlayingState] = useState(false); const [volume, setVolume] = useState(80); const [speed, setSpeed] = useState('1.0'); const [timeline, setTimeline] = useState({ elapsed: 0, total: 0 }); const [showToolbar, setShowToolbar] = useState(false); // Speaker Slots States (Voice Cloning) const [speakers, setSpeakers] = useState([ { id: 0, name: 'Alex', isRecording: false, voicePath: null, previewUrl: null, role: 'Primary Speaker', color: 'cyan' }, { id: 1, name: 'Sarah', isRecording: false, voicePath: null, previewUrl: null, role: 'Dialogue Partner', color: 'purple' } ]); // Master Export Hub State const [compiledAudioUrl, setCompiledAudioUrl] = useState(null); const [compiledAudioMode, setCompiledAudioMode] = useState(null); // 'stream' or 'file' const accumulatedSessionChunksRef = useRef([]); // Preview Voice Reference Playback State const [playingPreviewId, setPlayingPreviewId] = useState(null); const previewSourceRef = useRef(null); // Custom Naming Modal State const [namingModal, setNamingModal] = useState({ isOpen: false, fileBlob: null, speakerId: null, defaultName: '' }); const [voiceNameInput, setVoiceNameInput] = useState(''); // Custom Action (Rename/Delete) Modal State const [actionModal, setActionModal] = useState({ isOpen: false, type: '', // 'rename_voice', 'delete_voice', 'rename_podcast', 'delete_podcast' id: null, oldValue: '', inputValue: '', audioUrl: '', profileName: '' }); // Console Logs State const [logs, setLogs] = useState([ { type: 'system', text: 'Welcome to VibeVoice React Studio Console. System ready.', time: new Date().toLocaleTimeString() }, { type: 'info', text: 'VibeVoice neural weights loading directly on local CPU. Real-time zero-shot synthesis active.', time: new Date().toLocaleTimeString() } ]); // Web Audio Context & Playback References const audioCtxRef = useRef(null); const analyserNodeRef = useRef(null); const gainNodeRef = useRef(null); const wsRef = useRef(null); // Streaming Queue Refs (Gapless Playback) const playbackQueueRef = useRef([]); const nextPlayTimeRef = useRef(0); const isStreamingActiveRef = useRef(false); const scheduledSourcesRef = useRef([]); const doneReceivedRef = useRef(false); // Traditional REST Player Refs const traditionalSourceRef = useRef(null); const traditionalStartClockRef = useRef(0); const traditionalDurationRef = useRef(0); // Microphone Recording Refs const mediaRecordersRef = useRef([null, null]); const audioChunksRef = useRef([[], []]); // UI Canvas Visualizer Refs const canvasRef = useRef(null); // ========================================================================== // Console Log Helper // ========================================================================== const addLog = (type, text) => { setLogs(prev => [ ...prev, { type, text, time: new Date().toLocaleTimeString() } ]); }; // Scroll Console to Bottom useEffect(() => { const consoleBody = document.getElementById('consoleBody'); if (consoleBody) { consoleBody.scrollTop = consoleBody.scrollHeight; } }, [logs]); // ========================================================================== // Server Status & WebSocket Handshakes // ========================================================================== const checkServerStatus = async () => { try { const res = await fetch('/api/status'); const data = await res.json(); setModelStatus({ useRealModel: data.use_real_model, modelId: data.model_id, loaded: data.loaded, loading: data.loading, error: data.error, device: data.device }); } catch (e) { addLog('error', 'Failed to connect to VibeVoice backend API server.'); } }; const fetchSavedVoices = async () => { try { const { data, error } = await supabase.from('voice_profiles').select('*'); if (error) throw error; setSavedVoices(data || []); addLog('info', `Fetched ${data?.length || 0} saved voice profiles from Supabase.`); // Auto-assign matching saved voices to speakers by name to get "old audio input" instantly! if (data && data.length > 0) { setSpeakers(prev => prev.map(s => { const matched = data.find(v => v.name.trim().toLowerCase() === s.name.trim().toLowerCase()); if (matched && !s.voicePath) { addLog('info', `Auto-loaded saved voice profile "${matched.name}" into Speaker ${s.id} slot.`); return { ...s, voicePath: matched.audio_url, previewUrl: matched.audio_url }; } return s; })); } } catch (err) { addLog('error', 'Failed to fetch saved voice profiles from Supabase: ' + err.message); } }; const fetchPodcastHistory = async () => { try { const { data, error } = await supabase .from('podcast_history') .select('*') .order('created_at', { ascending: false }); if (error) throw error; setPodcastHistory(data || []); addLog('info', `Fetched ${data?.length || 0} archived podcasts from Supabase.`); } catch (err) { addLog('error', 'Failed to fetch podcast history: ' + err.message); } }; const savePodcastToHistory = async (wavBlob, textScript, mode) => { try { addLog('info', 'Archiving generated podcast to Supabase permanent storage...'); const fileName = `podcast_${Date.now()}.wav`; // 1. Upload to storage const { data: storageData, error: storageError } = await supabase .storage .from('voice_samples') .upload(fileName, wavBlob); if (storageError) throw storageError; // 2. Get public url const { data: publicUrlData } = supabase .storage .from('voice_samples') .getPublicUrl(fileName); const publicUrl = publicUrlData.publicUrl; // 3. Insert metadata record into DB const primaryVoice = speakers[0]?.voicePath || null; const matchingProfile = primaryVoice ? savedVoices.find(v => v.audio_url === primaryVoice) : null; const voiceProfileId = matchingProfile ? matchingProfile.id : null; const { data: dbData, error: dbError } = await supabase .from('podcast_history') .insert([{ title: `Podcast - ${new Date().toLocaleString()} (${mode})`, text_script: textScript, audio_url: publicUrl, speaker_id: 0, voice_profile_id: voiceProfileId }]) .select(); if (dbError) throw dbError; addLog('success', 'Podcast archived successfully to Supabase Database!'); fetchPodcastHistory(); } catch (err) { addLog('warning', `Failed to archive podcast to Supabase (${err.message}). Storing in local session memory...`); // Fallback: Create local preview URL for the compiled podcast WAV const localPreviewUrl = URL.createObjectURL(wavBlob); setPodcastHistory(prev => [ { id: `local_${Date.now()}`, title: `Podcast - ${new Date().toLocaleString()} (${mode}) [Local Session]`, text_script: textScript, audio_url: localPreviewUrl, speaker_id: 0, voice_profile_id: null, created_at: new Date().toISOString() }, ...prev ]); addLog('success', 'Podcast successfully cached in local session memory!'); } }; const saveTraditionalPodcastToHistory = async (audioUrl, textScript, mode) => { try { const response = await fetch(audioUrl); const blob = await response.blob(); await savePodcastToHistory(blob, textScript, mode); } catch (err) { addLog('error', 'Failed to download compiled audio for archiving: ' + err.message); } }; // --- Custom Action Modal Trigger Hooks --- const openRenameVoiceModal = (voiceId, oldName, audioUrl) => { setActionModal({ isOpen: true, type: 'rename_voice', id: voiceId, oldValue: oldName, inputValue: oldName, audioUrl, profileName: oldName }); }; const openDeleteVoiceModal = (voiceId, oldName, audioUrl) => { setActionModal({ isOpen: true, type: 'delete_voice', id: voiceId, oldValue: oldName, inputValue: '', audioUrl, profileName: oldName }); }; const openRenamePodcastModal = (podcastId, oldTitle) => { setActionModal({ isOpen: true, type: 'rename_podcast', id: podcastId, oldValue: oldTitle, inputValue: oldTitle, audioUrl: '', profileName: oldTitle }); }; const openDeletePodcastModal = (podcastId, title, audioUrl) => { setActionModal({ isOpen: true, type: 'delete_podcast', id: podcastId, oldValue: title, inputValue: '', audioUrl, profileName: title }); }; // --- Core execution functions invoked on Custom Modal Submit --- const executeRenamePodcast = async (podcastId, newTitle) => { if (!newTitle || newTitle.trim() === '') return; try { addLog('info', `Renaming podcast to "${newTitle.trim()}"...`); // Local podcast session if (typeof podcastId === 'string' && podcastId.startsWith('local_')) { setPodcastHistory(prev => prev.map(p => p.id === podcastId ? { ...p, title: newTitle.trim() } : p)); addLog('success', `Podcast successfully renamed to "${newTitle.trim()}" locally.`); return; } // Supabase DB podcast const { error } = await supabase .from('podcast_history') .update({ title: newTitle.trim() }) .eq('id', podcastId); if (error) throw error; addLog('success', `Podcast title successfully updated in Supabase Database!`); fetchPodcastHistory(); } catch (err) { addLog('error', 'Failed to rename podcast: ' + err.message); } }; const executeRenameVoice = async (voiceId, newName, audioUrl) => { if (!newName || newName.trim() === '') return; try { addLog('info', `Renaming voice profile to "${newName.trim()}"...`); // Local voice profiles if (typeof voiceId === 'string' && voiceId.startsWith('local_')) { setSavedVoices(prev => prev.map(v => v.id === voiceId ? { ...v, name: newName.trim() } : v)); // Update speakers setSpeakers(prev => prev.map(s => { if (s.voicePath === audioUrl) { return { ...s, name: newName.trim() }; } return s; })); addLog('success', `Voice profile successfully renamed to "${newName.trim()}" locally.`); return; } // Supabase const { error } = await supabase .from('voice_profiles') .update({ name: newName.trim() }) .eq('id', voiceId); if (error) throw error; addLog('success', `Voice profile successfully renamed in database!`); // Update state immediately setSavedVoices(prev => prev.map(v => v.id === voiceId ? { ...v, name: newName.trim() } : v)); // Also update any speaker slot using it setSpeakers(prev => prev.map(s => { if (s.voicePath === audioUrl) { return { ...s, name: newName.trim() }; } return s; })); } catch (err) { addLog('error', `Failed to rename voice profile: ${err.message}`); } }; const executeDeleteVoice = async (voiceId, audioUrl, profileName) => { try { addLog('info', `Deleting voice profile "${profileName}"...`); // Clear voice path of speakers who use this profile setSpeakers(prev => prev.map(s => s.voicePath === audioUrl ? { ...s, voicePath: null, previewUrl: null } : s)); // Local voice profile if (typeof voiceId === 'string' && (voiceId.startsWith('local_') || voiceId.endsWith('.wav'))) { setSavedVoices(prev => prev.filter(v => v.id !== voiceId)); if (audioUrl && audioUrl.startsWith('/static/cloned_voices/')) { const filename = audioUrl.split('/').pop(); try { await fetch(`/api/delete-voice/${filename}`, { method: 'DELETE' }); } catch(e) {} } addLog('success', `Voice profile "${profileName}" removed from local session.`); return; } // Database profile const { error: dbError } = await supabase .from('voice_profiles') .delete() .eq('id', voiceId); if (dbError) throw dbError; // Extract filename from URL and delete from storage if (audioUrl) { try { const filePart = audioUrl.split('/').pop(); if (filePart) { await supabase.storage.from('voice_samples').remove([filePart]); } } catch (storageErr) { console.warn('Failed to delete voice profile from Storage:', storageErr); } } addLog('success', `Voice profile "${profileName}" successfully deleted from Supabase.`); fetchSavedVoices(); } catch (err) { addLog('error', `Failed to delete voice profile: ${err.message}`); } }; const executeDeletePodcast = async (id, audioUrl, title) => { try { addLog('info', `Deleting podcast "${title}" from archive...`); if (typeof id === 'string' && id.startsWith('local_')) { setPodcastHistory(prev => prev.filter(p => p.id !== id)); addLog('success', 'Podcast deleted successfully from local session memory.'); return; } // 1. Delete from database const { error: dbError } = await supabase .from('podcast_history') .delete() .eq('id', id); if (dbError) throw dbError; // 2. Try to extract filename from URL and delete from Storage try { const filePart = audioUrl.split('/').pop(); if (filePart) { await supabase.storage.from('voice_samples').remove([filePart]); } } catch (err) { console.warn('Failed to delete audio file from Storage:', err); } addLog('success', 'Podcast deleted successfully.'); fetchPodcastHistory(); } catch (err) { addLog('error', 'Failed to delete podcast: ' + err.message); } }; const playHistoryPodcast = async (podcast) => { initAudio(); handleStopPlayback(); // Stop any other playing sessions try { addLog('info', `Streaming archived podcast: "${podcast.title}"...`); setIsAudioPlayingState(true); setCompiledAudioUrl(podcast.audio_url); setCompiledAudioMode('file'); loadTraditionalWav(podcast.audio_url); } catch (e) { addLog('error', 'Failed to play archived podcast.'); handleStopPlayback(); } }; useEffect(() => { addLog('system', 'Initializing VibeVoice Studio environment...'); checkServerStatus(); fetchSavedVoices(); fetchPodcastHistory(); // Poll server status periodically if loading const interval = setInterval(() => { checkServerStatus(); }, 4000); return () => clearInterval(interval); }, []); // WebSocket Connection Manager useEffect(() => { const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; const host = window.location.host; const wsUrl = window.location.hostname === 'localhost' || window.location.hostname === '127.0.0.1' ? 'ws://127.0.0.1:8000/api/stream' : `${protocol}//${host}/api/stream`; addLog('system', `Opening real-time WebSocket connection to ${wsUrl}`); const websocket = new WebSocket(wsUrl); wsRef.current = websocket; websocket.onopen = () => { setWsConnected(true); addLog('success', 'WebSocket real-time streaming link ESTABLISHED.'); }; websocket.onclose = () => { setWsConnected(false); addLog('error', 'WebSocket streaming link disconnected. Retrying connection...'); }; websocket.onerror = () => { addLog('error', 'WebSocket encountered connection error.'); }; websocket.onmessage = async (event) => { if (typeof event.data === 'string') { const data = JSON.parse(event.data); if (data.type === 'word') { addLog('word', `Agent spoke word: "${data.word}"`); } else if (data.type === 'done') { addLog('success', 'WebSocket voice generation chunk transfer complete.'); doneReceivedRef.current = true; // Generate local Object URL of the accumulated chunks for mastering if (accumulatedSessionChunksRef.current.length > 0) { try { const wavBlob = exportWav(accumulatedSessionChunksRef.current, SAMPLING_RATE); const objectUrl = URL.createObjectURL(wavBlob); setCompiledAudioUrl(objectUrl); setCompiledAudioMode('stream'); addLog('success', 'Master podcast output audio created and mastered for local export.'); // Automatically archive to Supabase history savePodcastToHistory(wavBlob, script, 'WebSocket Stream'); } catch (err) { addLog('error', `Failed to master session output: ${err.message}`); } } // If no sources are playing and queue is completely empty, finalize playback if (scheduledSourcesRef.current.length === 0 && playbackQueueRef.current.length === 0) { handleStopPlayback(); } } else if (data.type === 'error') { addLog('error', `Server stream error: ${data.message}`); handleStopPlayback(); } } else { // Raw Audio Binary Chunks const arrayBuffer = await event.data.arrayBuffer(); const int16Array = new Int16Array(arrayBuffer); const float32Array = new Float32Array(int16Array.length); for (let i = 0; i < int16Array.length; i++) { float32Array[i] = int16Array[i] / 32767.0; } if (isStreamingActiveRef.current) { playbackQueueRef.current.push(float32Array); // Accumulate for session download accumulatedSessionChunksRef.current.push(float32Array); scheduleNextStreamingChunk(); } } }; return () => { if (websocket) websocket.close(); }; }, []); // ========================================================================== // Web Audio Context & Node Setup (Pillar 2 & 3) // ========================================================================== const initAudio = () => { if (audioCtxRef.current) return; const AudioContextClass = window.AudioContext || window.webkitAudioContext; const context = new AudioContextClass({ sampleRate: SAMPLING_RATE }); audioCtxRef.current = context; const analyser = context.createAnalyser(); analyser.fftSize = 256; analyserNodeRef.current = analyser; const gain = context.createGain(); gain.gain.setValueAtTime(volume / 100, context.currentTime); gainNodeRef.current = gain; // Connect pipeline gain.connect(analyser); analyser.connect(context.destination); addLog('system', `Browser AudioContext engine running at ${SAMPLING_RATE}Hz.`); }; useEffect(() => { if (gainNodeRef.current && audioCtxRef.current) { gainNodeRef.current.gain.setValueAtTime(volume / 100, audioCtxRef.current.currentTime); } }, [volume]); // Streaming Gapless Playback Logic const startStreamingPlayback = () => { initAudio(); if (audioCtxRef.current.state === 'suspended') { audioCtxRef.current.resume(); } playbackQueueRef.current = []; accumulatedSessionChunksRef.current = []; // Clear previous export session chunks scheduledSourcesRef.current = []; doneReceivedRef.current = false; nextPlayTimeRef.current = audioCtxRef.current.currentTime + 0.15; isStreamingActiveRef.current = true; setIsAudioPlayingState(true); setAgentState({ status: 'speaking', title: 'STREAMING SPEECH', sub: 'Synthesizing voice chunks in real-time over WebSocket...' }); addLog('info', 'WebSocket player active. Buffering and playing...'); }; const scheduleNextStreamingChunk = () => { if (!isStreamingActiveRef.current || playbackQueueRef.current.length === 0) return; const chunk = playbackQueueRef.current.shift(); if (!chunk || chunk.length === 0) return; const context = audioCtxRef.current; const buffer = context.createBuffer(1, chunk.length, SAMPLING_RATE); buffer.copyToChannel(chunk, 0); const source = context.createBufferSource(); source.buffer = buffer; source.playbackRate.value = parseFloat(speed); source.connect(gainNodeRef.current); const startTime = Math.max(context.currentTime, nextPlayTimeRef.current); source.start(startTime); scheduledSourcesRef.current.push(source); const duration = buffer.duration / source.playbackRate.value; nextPlayTimeRef.current = startTime + duration; source.onended = () => { // Remove completed source scheduledSourcesRef.current = scheduledSourcesRef.current.filter(src => src !== source); // Check if queue is completely drained and done signal is received if (scheduledSourcesRef.current.length === 0 && playbackQueueRef.current.length === 0 && isStreamingActiveRef.current && doneReceivedRef.current) { handleStopPlayback(); addLog('info', 'Agent voice stream play completed.'); } }; }; const handleStopPlayback = () => { isStreamingActiveRef.current = false; playbackQueueRef.current = []; // Stop and clear all active buffers scheduledSourcesRef.current.forEach(src => { try { src.stop(); } catch(e) {} }); scheduledSourcesRef.current = []; if (traditionalSourceRef.current) { try { traditionalSourceRef.current.stop(); } catch(e) {} traditionalSourceRef.current = null; } // Stop reference preview playback if active if (previewSourceRef.current) { try { previewSourceRef.current.stop(); } catch(e) {} previewSourceRef.current = null; } setPlayingPreviewId(null); setIsAudioPlayingState(false); setShowToolbar(false); setAgentState({ status: 'idle', title: 'AGENT READY', sub: 'Configure scripts and trigger generation below' }); addLog('system', 'Agent audio playback stopped.'); }; // ========================================================================== // Canvas Waves Drawing Loop (AnalyserNode hook) // ========================================================================== useEffect(() => { const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext('2d'); const width = canvas.width; const height = canvas.height; let animationFrameId; const renderWave = () => { animationFrameId = requestAnimationFrame(renderWave); ctx.fillStyle = 'rgba(7, 9, 14, 0.25)'; // Back-sweep fade ctx.fillRect(0, 0, width, height); let dataArray = null; let bufferLength = 0; if (analyserNodeRef.current && isAudioPlayingState) { bufferLength = analyserNodeRef.current.frequencyBinCount; dataArray = new Uint8Array(bufferLength); analyserNodeRef.current.getByteTimeDomainData(dataArray); } ctx.lineWidth = 2.5; ctx.shadowBlur = isAudioPlayingState ? 10 : 0; ctx.shadowColor = 'rgba(0, 242, 254, 0.4)'; // Linear Neon color gradient across visualizer canvas const gradient = ctx.createLinearGradient(0, 0, width, 0); gradient.addColorStop(0, '#a259ff'); // Purple gradient.addColorStop(0.5, '#00f2fe'); // Cyan gradient.addColorStop(1, '#ff5c8d'); // Pink ctx.strokeStyle = gradient; ctx.beginPath(); if (dataArray) { // Draw real sound wave oscillations const sliceWidth = width / bufferLength; let x = 0; for (let i = 0; i < bufferLength; i++) { const v = dataArray[i] / 128.0; const y = (v * height) / 2; if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y); x += sliceWidth; } } else { // Draw calm animated breathing flatline when quiet const points = 40; const sliceWidth = width / points; let x = 0; const time = Date.now() * 0.003; for (let i = 0; i < points; i++) { const breathing = Math.sin(time) * 0.2 + 0.8; const y = height / 2 + Math.sin(i * 0.15 + time * 2) * (2.5 * breathing); if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y); x += sliceWidth; } } ctx.lineTo(width, height / 2); ctx.stroke(); }; renderWave(); return () => cancelAnimationFrame(animationFrameId); }, [isAudioPlayingState]); // ========================================================================== // Synthesis Triggers (REST vs WebSocket Stream) // ========================================================================== const handleSynthesize = async () => { const text = script.trim(); if (!text) { addLog('error', 'Cannot synthesize. Text script field is empty!'); return; } initAudio(); handleStopPlayback(); // Halt ongoing audio runs // Map dynamic speaker voice paths const speakerVoices = {}; speakers.forEach(s => { if (s.voicePath) { speakerVoices[s.id] = s.voicePath; } }); // Reset master download url setCompiledAudioUrl(null); setCompiledAudioMode(null); if (streamMode) { // WS Streaming Mode if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) { addLog('error', 'WebSocket is offline. Streaming mode disabled.'); return; } startStreamingPlayback(); wsRef.current.send(JSON.stringify({ text, voice_sample_path: speakers[0]?.voicePath || null, speaker_id: 0, speaker_voices: speakerVoices })); } else { // Traditional File-based Playback Mode try { addLog('info', 'Compiling voice script... Sending POST to VibeVoice generator...'); setAgentState({ status: 'thinking', title: 'COMPILING AUDIO', sub: 'Running neural speech generation weights...' }); setIsAudioPlayingState(true); const res = await fetch('/api/generate', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ text, voice_sample_path: speakers[0]?.voicePath || null, speaker_id: 0, speaker_voices: speakerVoices }) }); const data = await res.json(); if (data.status === 'success') { addLog('success', `Voice file compiled. Method: [${data.mode}]. Playing track...`); loadTraditionalWav(data.audio_url); setCompiledAudioUrl(data.audio_url); setCompiledAudioMode('file'); // Automatically fetch compiled file and archive to Supabase history saveTraditionalPodcastToHistory(data.audio_url, text, data.mode); } else { throw new Error('Server returned error response.'); } } catch (e) { handleStopPlayback(); addLog('error', `Synthesis failed: ${e.message}`); } } }; const loadTraditionalWav = async (audioUrl) => { try { const res = await fetch(audioUrl); const arrayBuffer = await res.arrayBuffer(); audioCtxRef.current.decodeAudioData(arrayBuffer, (decodedBuffer) => { const source = audioCtxRef.current.createBufferSource(); source.buffer = decodedBuffer; source.playbackRate.value = parseFloat(speed); source.connect(gainNodeRef.current); source.start(0); traditionalSourceRef.current = source; traditionalDurationRef.current = decodedBuffer.duration; traditionalStartClockRef.current = audioCtxRef.current.currentTime; setAgentState({ status: 'speaking', title: 'PLAYING FILE', sub: 'Streaming synthesized high-fidelity compiled audio track...' }); setShowToolbar(true); setTimeline({ elapsed: 0, total: decodedBuffer.duration }); const tickTimeline = () => { if (traditionalSourceRef.current === source) { const elapsed = (audioCtxRef.current.currentTime - traditionalStartClockRef.current) * parseFloat(speed); setTimeline(prev => ({ ...prev, elapsed: Math.min(elapsed, decodedBuffer.duration) })); if (elapsed < decodedBuffer.duration) { requestAnimationFrame(tickTimeline); } } }; tickTimeline(); source.onended = () => { if (traditionalSourceRef.current === source) { handleStopPlayback(); } }; }, (err) => { addLog('error', 'Failed to decode WAV file buffer.'); handleStopPlayback(); }); } catch(e) { addLog('error', 'Error playing generated track.'); handleStopPlayback(); } }; const formatTime = (seconds) => { const m = Math.floor(seconds / 60).toString().padStart(2, '0'); const s = Math.floor(seconds % 60).toString().padStart(2, '0'); return `${m}:${s}`; }; const encodeWAV = (samples, sampleRate) => { const buffer = new ArrayBuffer(44 + samples.length * 2); const view = new DataView(buffer); const writeString = (view, offset, string) => { for (let i = 0; i < string.length; i++) { view.setUint8(offset + i, string.charCodeAt(i)); } }; const floatTo16BitPCM = (output, offset, input) => { for (let i = 0; i < input.length; i++, offset += 2) { let s = Math.max(-1, Math.min(1, input[i])); output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true); } }; /* RIFF identifier */ writeString(view, 0, 'RIFF'); /* file length */ view.setUint32(4, 36 + samples.length * 2, true); /* RIFF type */ writeString(view, 8, 'WAVE'); /* format chunk identifier */ writeString(view, 12, 'fmt '); /* format chunk length */ view.setUint32(16, 16, true); /* sample format (raw) */ view.setUint16(20, 1, true); /* channel count */ view.setUint16(22, 1, true); /* sample rate */ view.setUint32(24, sampleRate, true); /* byte rate (sample rate * block align) */ view.setUint32(28, sampleRate * 2, true); /* block align (channel count * bytes per sample) */ view.setUint16(32, 2, true); /* bits per sample */ view.setUint16(34, 16, true); /* data chunk identifier */ writeString(view, 36, 'data'); /* data chunk length */ view.setUint32(40, samples.length * 2, true); floatTo16BitPCM(view, 44, samples); return new Blob([view], { type: 'audio/wav' }); }; // ========================================================================== // Voice Cloning Microphone Hooks (Pillar 4) // ========================================================================== const toggleRecording = async (speakerId) => { const speaker = speakers.find(s => s.id === speakerId); if (!speaker) return; if (speaker.isRecording) { // Stop recording const recorder = mediaRecordersRef.current[speakerId]; if (recorder) { recorder.stop(); setSpeakers(prev => prev.map(s => s.id === speakerId ? { ...s, isRecording: false } : s)); setAgentState({ status: 'idle', title: 'AGENT READY', sub: 'Voice profile recorded. Syncing to neural engine...' }); } } else { // Start recording try { addLog('info', `Requesting mic permission for Speaker ${speakerId} (${speaker.name})...`); const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); setAgentState({ status: 'listening', title: 'LISTENING...', sub: 'Speak normally for 10-15s to capture vocal signature...' }); // Use independent default sample-rate AudioContext specifically for mic capture to avoid browser resampling bugs const RecordingContextClass = window.AudioContext || window.webkitAudioContext; const recContext = new RecordingContextClass(); const source = recContext.createMediaStreamSource(stream); const processor = recContext.createScriptProcessor(4096, 1, 1); const recordingSamples = []; processor.onaudioprocess = (e) => { const channelData = e.inputBuffer.getChannelData(0); recordingSamples.push(new Float32Array(channelData)); }; source.connect(processor); processor.connect(recContext.destination); mediaRecordersRef.current[speakerId] = { stop: () => { source.disconnect(processor); processor.disconnect(recContext.destination); stream.getTracks().forEach(track => track.stop()); recContext.close(); addLog('info', 'Encoding microphone speech data to 16-bit PCM WAV...'); let totalLength = 0; for (let i = 0; i < recordingSamples.length; i++) { totalLength += recordingSamples[i].length; } const flattened = new Float32Array(totalLength); let offset = 0; for (let i = 0; i < recordingSamples.length; i++) { flattened.set(recordingSamples[i], offset); offset += recordingSamples[i].length; } const wavBlob = encodeWAV(flattened, recContext.sampleRate); setNamingModal({ isOpen: true, fileBlob: wavBlob, speakerId: speakerId, defaultName: speaker.name }); setVoiceNameInput(speaker.name); } }; setSpeakers(prev => prev.map(s => s.id === speakerId ? { ...s, isRecording: true } : s)); addLog('success', 'Microphone recording started. Speak clearly now!'); } catch (e) { addLog('error', `Microphone recording failed: ${e.message}`); setAgentState({ status: 'idle', title: 'AGENT READY', sub: 'Record trigger failed or permission denied' }); } } }; const handleFileUpload = (e, speakerId) => { const file = e.target.files[0]; if (!file) return; const speaker = speakers.find(s => s.id === speakerId); addLog('info', `Selected file for upload: ${file.name}`); const cleanFileName = file.name.replace(/\.[^/.]+$/, ""); setNamingModal({ isOpen: true, fileBlob: file, speakerId: speakerId, defaultName: cleanFileName }); setVoiceNameInput(cleanFileName); }; // Play/Stop Reference Voice Preview using main AudioContext pipeline const playPreview = async (speakerId, url) => { initAudio(); handleStopPlayback(); // Stop any other playing session if (playingPreviewId === speakerId) { stopPreviewPlayback(); return; } try { addLog('info', `Playing voice preview for Speaker ${speakerId}...`); setIsAudioPlayingState(true); setPlayingPreviewId(speakerId); const res = await fetch(url); const arrayBuffer = await res.arrayBuffer(); audioCtxRef.current.decodeAudioData(arrayBuffer, (decodedBuffer) => { const source = audioCtxRef.current.createBufferSource(); source.buffer = decodedBuffer; source.connect(gainNodeRef.current); source.start(0); previewSourceRef.current = source; setAgentState({ status: 'speaking', title: 'PLAYING REFERENCE', sub: `Listening to cloned voice reference profile...` }); source.onended = () => { stopPreviewPlayback(); }; }, (err) => { addLog('error', 'Failed to decode reference audio file buffer.'); stopPreviewPlayback(); }); } catch (e) { addLog('error', 'Error playing reference voice.'); stopPreviewPlayback(); } }; const stopPreviewPlayback = () => { if (previewSourceRef.current) { try { previewSourceRef.current.stop(); } catch(e) {} previewSourceRef.current = null; } setPlayingPreviewId(null); setIsAudioPlayingState(false); setAgentState({ status: 'idle', title: 'AGENT READY', sub: 'Configure scripts and trigger generation below' }); }; // Add Dynamic Speaker Slot const addSpeaker = () => { const nextId = speakers.length > 0 ? Math.max(...speakers.map(s => s.id)) + 1 : 0; const speakerColors = ['cyan', 'purple', 'emerald', 'amber', 'rose', 'blue']; const color = speakerColors[nextId % speakerColors.length]; const roles = ['Podcast Guest', 'Narrator', 'Secondary Host', 'Panelist', 'Expert Analyst']; const role = roles[(nextId - 2) % roles.length] || 'Guest Slot'; const newSpk = { id: nextId, name: `Speaker ${nextId}`, isRecording: false, voicePath: null, previewUrl: null, role: role, color: color }; setSpeakers(prev => [...prev, newSpk]); addLog('system', `Added dynamic speaker slot: [Speaker ${nextId}]`); }; // Remove Dynamic Speaker Slot const removeSpeaker = (speakerId) => { if (speakerId === 0) { addLog('error', 'Cannot delete Speaker 0 (Primary Speaker is required).'); return; } handleStopPlayback(); setSpeakers(prev => prev.filter(s => s.id !== speakerId)); addLog('system', `Removed speaker slot: [Speaker ${speakerId}]`); }; const uploadVoiceProfile = async (fileBlob, speakerId, speakerName) => { try { addLog('info', `Uploading permanent voice profile "${speakerName}" to Supabase...`); const fileName = `voice_${Date.now()}_${speakerId}.wav`; // 1. Upload to Supabase Storage const { data: storageData, error: storageError } = await supabase .storage .from('voice_samples') .upload(fileName, fileBlob); if (storageError) throw storageError; // Get public URL const { data: publicUrlData } = supabase .storage .from('voice_samples') .getPublicUrl(fileName); const publicUrl = publicUrlData.publicUrl; // 2. Insert into Supabase Database const { data: dbData, error: dbError } = await supabase .from('voice_profiles') .insert([{ name: speakerName, audio_url: publicUrl }]) .select(); if (dbError) throw dbError; // 3. Update UI state setSpeakers(prev => prev.map(s => s.id === speakerId ? { ...s, voicePath: publicUrl, previewUrl: publicUrl } : s)); addLog('success', `Voice signature for "${speakerName}" saved to Supabase successfully!`); // Refresh the saved voices list fetchSavedVoices(); } catch(e) { addLog('warning', `Supabase upload failed (${e.message}). Falling back to local backend storage...`); try { const formData = new FormData(); const fileToUpload = fileBlob instanceof File ? fileBlob : new File([fileBlob], `voice_${Date.now()}_${speakerId}.wav`, { type: 'audio/wav' }); formData.append('file', fileToUpload); formData.append('speaker_name', speakerName); const res = await fetch('/api/upload-voice', { method: 'POST', body: formData }); if (!res.ok) throw new Error(`Backend returned status ${res.status}`); const data = await res.json(); if (data.status === 'success') { const localUrl = '/' + data.voice_path; setSpeakers(prev => prev.map(s => s.id === speakerId ? { ...s, voicePath: localUrl, previewUrl: localUrl } : s)); addLog('success', `Voice signature for "${speakerName}" saved locally on server!`); // Instantly add this local voice to the savedVoices dropdown selection setSavedVoices(prev => { const exists = prev.some(v => v.audio_url === localUrl); if (exists) return prev; return [...prev, { id: data.filename, name: speakerName, audio_url: localUrl }]; }); } else { throw new Error(data.message || 'Local upload failed'); } } catch (localErr) { addLog('error', `Voice signature local upload fallback failed: ${localErr.message}`); } } }; // Toggle AI Model Mode on Backend const handleToggleModelMode = async () => { const enable = !modelStatus.useRealModel; setTogglingModel(true); addLog('info', `Requesting engine toggle: ${enable ? 'Enabling AI Model' : 'Enabling Demo Synthesizer'}`); try { const res = await fetch(`/api/toggle-model?enable=${enable}`, { method: 'POST' }); const data = await res.json(); setModelStatus(prev => ({ ...prev, useRealModel: data.use_real_model, loading: enable, // starts load loaded: false })); setTimeout(checkServerStatus, 1500); } catch (e) { addLog('error', 'Toggle engine request failed.'); } finally { setTogglingModel(false); } }; // Presets loaders const loadPreset = (type) => { if (type === 'podcast') { setScript(`Speaker 0: Welcome back to the Vibe Voice podcast segment. Today we are cloning human expressions. Speaker 1: That is right! The 0.5B model manages this dialogue transition on a local CPU extremely fast. Speaker 0: Incredible! It sounds like two real people sitting in this glassmorphic studio together.`); addLog('info', 'Loaded template: [Podcast Dialogue Interview]'); } else if (type === 'assistant') { setScript("Speaker 0: Hello, I am your personalized voice assistant. How can I help you customize your Dine Direct orders today?"); addLog('info', 'Loaded template: [DineDirect AI Support Agent]'); } else if (type === 'news') { setScript("Speaker 1: Breaking news from Microsoft Research. The Vibe Voice speech synthesis models are now live on local devices."); addLog('info', 'Loaded template: [News Bulletins Broadcast]'); } }; if (currentView === 'landing') { return setCurrentView('studio')} />; } return (
{/* Background Floating Ambient Blobs */}
{/* Header Widget */}
setCurrentView('landing')}>

VibeVoiceStudio

Microsoft Frontier Speech Synthesis

{/* Engine Status Panel */}
VibeVoice Engine: {modelStatus.loading ? 'Loading AI...' : 'VibeVoice-1.5 AI'}
{/* Connection badge */}
{wsConnected ? 'Connected' : 'Offline'}
{/* Dashboard Grid Grid */}
{/* Left Column: Voice Cloning Controls */}

Voice Cloning Center

Clone any voice in seconds. Record or upload a 10-30s audio sample to create a speaker profile prompt.

{/* Speaker Slots */} {speakers.map((spk, idx) => { const colorClasses = spk.id === 0 ? 'bg-[#00f2fe]/6 border-[#00f2fe]/20 text-[#00f2fe]' : spk.id === 1 ? 'bg-[#a259ff]/6 border-[#a259ff]/20 text-[#a259ff]' : spk.color === 'emerald' ? 'bg-[#00e673]/6 border-[#00e673]/20 text-[#00e673]' : spk.color === 'amber' ? 'bg-[#ffb703]/6 border-[#ffb703]/20 text-[#ffb703]' : spk.color === 'rose' ? 'bg-[#ff5c8d]/6 border-[#ff5c8d]/20 text-[#ff5c8d]' : 'bg-[#00f2fe]/6 border-[#00f2fe]/20 text-[#00f2fe]'; return (
Speaker {spk.id} {spk.id > 0 && ( )}
{spk.role}
{ const val = e.target.value; setSpeakers(prev => prev.map(s => s.id === spk.id ? { ...s, name: val } : s)); }} className="bg-black/20 border border-white/6 rounded-lg px-3 py-1.5 text-xs text-[#f0f3fa] focus:border-[#00f2fe] focus:shadow-[0_0_8px_rgba(0,242,254,0.15)] outline-none" />
{spk.voicePath && (() => { const matchedVoice = savedVoices.find(v => v.audio_url === spk.voicePath); if (matchedVoice) { return (
); } return null; })()}
{spk.previewUrl && (
Voice Cloned Successfully {/* Premium Custom Reference Audio Player */}
reference_{spk.name}.wav Ready for zero-shot synthesis
)}
); })} {/* Add Speaker Slot Trigger */} {/* Presets */}

Dialogue Templates

{/* 🎙️ Voice Vault (Saved Neural Profiles) */}

Voice Vault ({savedVoices.length})

Storage
{savedVoices.length === 0 ? (

No voice profiles saved yet.

) : ( savedVoices.map(v => (
{v.name}
)) )}
{/* Right Column: Execution Panels */}
{/* Visualizer Container */}
{/* Visual pulsating holographic Orb */}
{/* Glowing backing shadows */}
{/* Expansion speech rings */} {agentState.status === 'speaking' && ( <>
)}
{agentState.title}

{agentState.sub}

{/* Waveform Canvas */}
{/* Script Studio Box */}

Podcast Script Studio

{/* Real-time Streaming Check */}
{/* Textarea script fields */}