// Global Variables let mediaRecorder = null; let audioChunks = []; let isRecording = false; let recordingStartTime = null; let timerInterval = null; let currentAudio = null; let responseLanguage = 'en'; // 'en' for English only, 'si-en' for Sinhala+English // DOM Elements - Voice Chat const micBtn = document.getElementById('micBtn'); const statusIndicator = document.getElementById('statusIndicator'); const statusDot = statusIndicator.querySelector('.status-dot'); const statusText = statusIndicator.querySelector('.status-text'); const recordingTimer = document.getElementById('recordingTimer'); const timerText = recordingTimer.querySelector('.timer-text'); const visualizer = document.getElementById('visualizer'); const userText = document.getElementById('userText'); const botText = document.getElementById('botText'); const speakerBtn = document.getElementById('speakerBtn'); const pauseBtn = document.getElementById('pauseBtn'); const loadingOverlay = document.getElementById('loadingOverlay'); const loadingText = document.getElementById('loadingText'); const chatContainer = document.getElementById('chatContainer'); const resetBtn = document.getElementById('resetBtn'); // DOM Elements - Sections const voiceChatSection = document.getElementById('voiceChatSection'); // Initialize document.addEventListener('DOMContentLoaded', () => { checkBrowserSupport(); setupEventListeners(); }); // Check browser support for audio recording function checkBrowserSupport() { if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { showError('Your browser does not support audio recording. Please use a modern browser like Chrome or Firefox.'); micBtn.disabled = true; } } // Setup Event Listeners function setupEventListeners() { micBtn.addEventListener('click', toggleRecording); speakerBtn.addEventListener('click', playResponse); // Pause button if (pauseBtn) { pauseBtn.addEventListener('click', pauseAudio); } // Reset button - also clears history if (resetBtn) { resetBtn.addEventListener('click', resetRecording); } } // Toggle Recording async function toggleRecording() { if (isRecording) { stopRecording(); } else { await startRecording(); } } // Start Recording async function startRecording() { try { const stream = await navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true } }); // Determine the best supported MIME type let mimeType = 'audio/webm'; if (MediaRecorder.isTypeSupported('audio/webm;codecs=opus')) { mimeType = 'audio/webm;codecs=opus'; } else if (MediaRecorder.isTypeSupported('audio/webm')) { mimeType = 'audio/webm'; } else if (MediaRecorder.isTypeSupported('audio/mp4')) { mimeType = 'audio/mp4'; } else if (MediaRecorder.isTypeSupported('audio/ogg')) { mimeType = 'audio/ogg'; } mediaRecorder = new MediaRecorder(stream, { mimeType }); audioChunks = []; mediaRecorder.ondataavailable = (event) => { if (event.data.size > 0) { audioChunks.push(event.data); } }; mediaRecorder.onstop = async () => { const audioBlob = new Blob(audioChunks, { type: mimeType }); stream.getTracks().forEach(track => track.stop()); await processAudio(audioBlob); }; mediaRecorder.start(100); // Collect data every 100ms isRecording = true; recordingStartTime = Date.now(); // Update UI updateUIForRecording(true); startTimer(); } catch (error) { console.error('Error starting recording:', error); showError('Could not access microphone. Please allow microphone permission.'); } } // Stop Recording function stopRecording() { if (mediaRecorder && mediaRecorder.state !== 'inactive') { mediaRecorder.stop(); isRecording = false; stopTimer(); updateUIForRecording(false); } } // Update UI for Recording State function updateUIForRecording(recording) { if (recording) { micBtn.classList.add('recording'); statusDot.classList.add('recording'); statusText.textContent = 'Recording...'; recordingTimer.classList.add('active'); visualizer.classList.add('active'); } else { micBtn.classList.remove('recording'); statusDot.classList.remove('recording'); statusText.textContent = 'Processing...'; recordingTimer.classList.remove('active'); visualizer.classList.remove('active'); } } // Timer Functions function startTimer() { timerInterval = setInterval(() => { const elapsed = Date.now() - recordingStartTime; const minutes = Math.floor(elapsed / 60000); const seconds = Math.floor((elapsed % 60000) / 1000); timerText.textContent = `${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')}`; }, 100); } function stopTimer() { if (timerInterval) { clearInterval(timerInterval); timerInterval = null; } timerText.textContent = '00:00'; } // Process Audio - Send to Backend async function processAudio(audioBlob) { showLoading('Converting speech to text...'); try { // Convert to WAV format for better compatibility const wavBlob = await convertToWav(audioBlob); // Create form data const formData = new FormData(); formData.append('audio', wavBlob, 'recording.wav'); // Send to speech-to-text endpoint const sttResponse = await fetch('/api/speech-to-text', { method: 'POST', body: formData }); if (!sttResponse.ok) { const error = await sttResponse.json(); throw new Error(error.detail || 'Speech recognition failed'); } const sttResult = await sttResponse.json(); const transcribedText = sttResult.text; // Show original transcription temporarily displayUserText(transcribedText + ' (translating...)'); // Step 2: Translate to English showLoading('Translating to English...'); let englishText = transcribedText; let translationSuccess = false; try { const translateRes = await fetch('/api/translate-to-english', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ question: transcribedText }) }); if (translateRes.ok) { const translateData = await translateRes.json(); if (translateData.translated && translateData.english_question) { englishText = translateData.english_question; translationSuccess = true; } else if (translateData.english_question && translateData.english_question !== transcribedText) { // Even if translated flag is false, check if we got different text englishText = translateData.english_question; translationSuccess = true; } } } catch (translateError) { console.error('Translation error:', translateError); } // Display both original and English if translation succeeded, otherwise just show original if (translationSuccess && englishText !== transcribedText) { displayUserTextWithOriginal(transcribedText, englishText); } else { displayUserText(transcribedText + ' (translation failed - using original)'); } // Step 3: Use RAG first, fallback to Gemini API showLoading('Searching knowledge base...'); const ragResponse = await fetch('/api/rag/ask', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ question: englishText, response_lang: responseLanguage // 'en' or 'si-en' }) }); if (!ragResponse.ok) { const error = await ragResponse.json(); throw new Error(error.detail || 'Query failed'); } const ragResult = await ragResponse.json(); const botResponse = ragResult.answer; const source = ragResult.source; // 'rag', 'gemini', or 'none' // Display bot response with source indicator displayBotTextWithSource(botResponse, source); // Enable speaker button speakerBtn.disabled = false; // Update status updateStatus('ready', 'Ready'); } catch (error) { console.error('Processing error:', error); showError(error.message); updateStatus('ready', 'Ready'); } finally { hideLoading(); } } // Convert audio blob to WAV format async function convertToWav(audioBlob) { return new Promise((resolve, reject) => { const audioContext = new (window.AudioContext || window.webkitAudioContext)(); const reader = new FileReader(); reader.onload = async () => { try { const arrayBuffer = reader.result; const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); // Resample to 16kHz for Whisper model const targetSampleRate = 16000; const offlineContext = new OfflineAudioContext( 1, // mono audioBuffer.duration * targetSampleRate, targetSampleRate ); const source = offlineContext.createBufferSource(); source.buffer = audioBuffer; source.connect(offlineContext.destination); source.start(0); const renderedBuffer = await offlineContext.startRendering(); const wavBlob = audioBufferToWav(renderedBuffer); resolve(wavBlob); } catch (error) { // If conversion fails, return original blob console.warn('WAV conversion failed, using original format:', error); resolve(audioBlob); } }; reader.onerror = () => reject(reader.error); reader.readAsArrayBuffer(audioBlob); }); } // Convert AudioBuffer to WAV Blob function audioBufferToWav(buffer) { const numChannels = buffer.numberOfChannels; const sampleRate = buffer.sampleRate; const format = 1; // PCM const bitDepth = 16; const bytesPerSample = bitDepth / 8; const blockAlign = numChannels * bytesPerSample; const dataLength = buffer.length * blockAlign; const bufferLength = 44 + dataLength; const arrayBuffer = new ArrayBuffer(bufferLength); const view = new DataView(arrayBuffer); // WAV header writeString(view, 0, 'RIFF'); view.setUint32(4, 36 + dataLength, true); writeString(view, 8, 'WAVE'); writeString(view, 12, 'fmt '); view.setUint32(16, 16, true); view.setUint16(20, format, true); view.setUint16(22, numChannels, true); view.setUint32(24, sampleRate, true); view.setUint32(28, sampleRate * blockAlign, true); view.setUint16(32, blockAlign, true); view.setUint16(34, bitDepth, true); writeString(view, 36, 'data'); view.setUint32(40, dataLength, true); // Write audio data const channelData = buffer.getChannelData(0); let offset = 44; for (let i = 0; i < channelData.length; i++) { const sample = Math.max(-1, Math.min(1, channelData[i])); view.setInt16(offset, sample < 0 ? sample * 0x8000 : sample * 0x7FFF, true); offset += 2; } return new Blob([arrayBuffer], { type: 'audio/wav' }); } function writeString(view, offset, string) { for (let i = 0; i < string.length; i++) { view.setUint8(offset + i, string.charCodeAt(i)); } } // Display Functions function displayUserText(text) { userText.innerHTML = `

${escapeHtml(text)}

`; } function displayUserTextWithOriginal(originalText, englishText) { userText.innerHTML = `

${escapeHtml(originalText)}

`; } function displayBotText(text) { // Convert markdown-like formatting to HTML const formattedText = formatText(text); botText.innerHTML = formattedText; } function displayBotTextWithSource(text, source) { // Convert markdown-like formatting to HTML with source badge const formattedText = formatText(text); let sourceLabel = ''; if (source === 'rag') { sourceLabel = ' From Documents'; } else if (source === 'gemini') { sourceLabel = ' From AI'; } botText.innerHTML = sourceLabel + formattedText; } function formatText(text) { // Basic formatting let formatted = escapeHtml(text); // Convert line breaks formatted = formatted.replace(/\n/g, '
'); // Convert **bold** to formatted = formatted.replace(/\*\*(.*?)\*\*/g, '$1'); // Convert *italic* to formatted = formatted.replace(/\*(.*?)\*/g, '$1'); return `

${formatted}

`; } function escapeHtml(text) { const div = document.createElement('div'); div.textContent = text; return div.innerHTML; } // Play Response using TTS async function playResponse() { const text = botText.textContent || botText.innerText; if (!text || text.includes('will appear here')) { return; } // If paused, resume if (currentAudio && currentAudio.paused) { currentAudio.play(); speakerBtn.classList.add('playing'); pauseBtn.classList.remove('paused'); pauseBtn.querySelector('i').className = 'fas fa-pause'; return; } // Stop current audio if playing if (currentAudio) { currentAudio.pause(); currentAudio = null; speakerBtn.classList.remove('playing'); } speakerBtn.classList.add('playing'); speakerBtn.querySelector('i').className = 'fas fa-spinner fa-spin'; try { const ttsLang = responseLanguage === 'en' ? 'en' : 'si'; const response = await fetch('/api/text-to-speech', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ text: text, lang: ttsLang }) }); if (!response.ok) { throw new Error('Text-to-speech failed'); } const audioBlob = await response.blob(); const audioUrl = URL.createObjectURL(audioBlob); currentAudio = new Audio(audioUrl); currentAudio.onended = () => { speakerBtn.classList.remove('playing'); speakerBtn.querySelector('i').className = 'fas fa-volume-up'; pauseBtn.classList.remove('paused'); pauseBtn.querySelector('i').className = 'fas fa-pause'; URL.revokeObjectURL(audioUrl); currentAudio = null; }; currentAudio.onerror = () => { speakerBtn.classList.remove('playing'); speakerBtn.querySelector('i').className = 'fas fa-volume-up'; showError('Failed to play audio'); }; await currentAudio.play(); speakerBtn.querySelector('i').className = 'fas fa-volume-up'; } catch (error) { console.error('TTS error:', error); speakerBtn.classList.remove('playing'); speakerBtn.querySelector('i').className = 'fas fa-volume-up'; showError('Text-to-speech failed'); } } // Pause Audio Playback function pauseAudio() { if (currentAudio && !currentAudio.paused) { currentAudio.pause(); speakerBtn.classList.remove('playing'); pauseBtn.classList.add('paused'); pauseBtn.querySelector('i').className = 'fas fa-play'; } else if (currentAudio && currentAudio.paused) { currentAudio.play(); speakerBtn.classList.add('playing'); pauseBtn.classList.remove('paused'); pauseBtn.querySelector('i').className = 'fas fa-pause'; } } // Reset Recording / Stop current action function resetRecording() { if (isRecording) { stopRecording(); } if (currentAudio) { currentAudio.pause(); currentAudio = null; speakerBtn.classList.remove('playing'); } updateStatus('ready', 'Ready'); clearHistory(); } // Clear Conversation History async function clearHistory() { try { const response = await fetch('/api/clear-history', { method: 'POST' }); if (response.ok) { // Reset UI userText.innerHTML = '

Your transcribed message will appear here...

'; botText.innerHTML = '

Bot response will appear here...

'; speakerBtn.disabled = true; // Show confirmation showSuccess('Conversation history cleared'); } } catch (error) { console.error('Error clearing history:', error); showError('Failed to clear history'); } } // Loading Functions function showLoading(message = 'Processing...') { loadingText.textContent = message; loadingOverlay.classList.add('active'); } function hideLoading() { loadingOverlay.classList.remove('active'); } // Status Update function updateStatus(state, text) { statusDot.className = 'status-dot'; if (state !== 'ready') { statusDot.classList.add(state); } statusText.textContent = text; } // Notification Functions function showError(message) { // Create toast notification showToast(message, 'error'); // Clear user and bot input fields after 2 seconds setTimeout(() => { if (userText) { userText.innerHTML = '

Your transcribed message will appear here...

'; } if (botText) { botText.innerHTML = '

Bot response will appear here...

'; } }, 2000); } function showSuccess(message) { showToast(message, 'success'); } function showToast(message, type = 'info') { // Remove existing toasts const existingToasts = document.querySelectorAll('.toast'); existingToasts.forEach(t => t.remove()); // Create toast element const toast = document.createElement('div'); toast.className = `toast toast-${type}`; toast.innerHTML = ` ${message} `; // Add styles toast.style.cssText = ` position: fixed; bottom: 20px; left: 50%; transform: translateX(-50%); padding: 12px 24px; background: ${type === 'error' ? '#ef4444' : '#22c55e'}; color: white; border-radius: 8px; display: flex; align-items: center; gap: 10px; z-index: 2000; box-shadow: 0 4px 20px rgba(0,0,0,0.3); animation: slideUp 0.3s ease; `; // Add animation keyframes if not exists if (!document.getElementById('toast-styles')) { const style = document.createElement('style'); style.id = 'toast-styles'; style.textContent = ` @keyframes slideUp { from { transform: translateX(-50%) translateY(100%); opacity: 0; } to { transform: translateX(-50%) translateY(0); opacity: 1; } } `; document.head.appendChild(style); } document.body.appendChild(toast); // Remove after 4 seconds setTimeout(() => { toast.style.opacity = '0'; toast.style.transition = 'opacity 0.3s ease'; setTimeout(() => toast.remove(), 300); }, 4000); }