/** * app.js — Main frontend orchestration using Gradio JS Client. */ import { Client } from "https://cdn.jsdelivr.net/npm/@gradio/client/dist/index.min.js"; import { startWaveform, stopWaveform } from './waveform.js'; import { renderTriageCard } from './triage-card.js'; let gradioClient = null; // --- DOM Elements --- const statusPill = document.getElementById('status-pill'); const statusText = statusPill.querySelector('.status-text'); // Mic Card const micBtn = document.getElementById('mic-button'); const micStatus = document.getElementById('mic-status'); const waveformCanvas = document.getElementById('waveform-canvas'); const transcriptContainer = document.getElementById('transcript-container'); const transcriptText = document.getElementById('transcript-text'); const editTranscriptBtn = document.getElementById('edit-transcript-btn'); // Camera Card const openCameraBtn = document.getElementById('open-camera-btn'); const uploadPhotoBtn = document.getElementById('upload-photo-btn'); const captureBtn = document.getElementById('capture-btn'); const fileInput = document.getElementById('photo-file-input'); const cameraPreview = document.getElementById('camera-preview'); const photoPreview = document.getElementById('photo-preview'); const cameraPlaceholder = document.getElementById('camera-placeholder'); const photoDescriptionBox = document.getElementById('photo-description'); // Triage & Output const triageBtn = document.getElementById('triage-btn'); const outputSection = document.getElementById('output-section'); const triageCardContainer = document.getElementById('triage-card'); const resetBtn = document.getElementById('reset-btn'); // --- Global State --- let isRecording = false; let mediaRecorder = null; let audioChunks = []; let waveformCleanup = null; let currentCameraStream = null; let state = { symptomsText: "", visualDescription: "", hindiAudioB64: "", englishAudioB64: "" }; // --- Initialization --- async function initClient() { try { gradioClient = await Client.connect(window.location.origin); statusPill.setAttribute('data-state', 'ready'); statusText.textContent = "Ready"; // Ping health endpoint via fetch (just to verify) fetch('/health').catch(e => console.error("Health check failed", e)); } catch (err) { console.error("Gradio Client connection failed:", err); statusPill.setAttribute('data-state', 'error'); statusText.textContent = "Offline"; } } document.addEventListener('DOMContentLoaded', initClient); // --- Audio Recording Flow --- micBtn.addEventListener('click', async () => { if (!gradioClient) { alert("System is still connecting. Please wait."); return; } if (!isRecording) { try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); mediaRecorder = new MediaRecorder(stream); audioChunks = []; mediaRecorder.ondataavailable = event => { if (event.data.size > 0) audioChunks.push(event.data); }; mediaRecorder.onstop = async () => { const audioBlob = new Blob(audioChunks, { type: 'audio/webm' }); stream.getTracks().forEach(track => track.stop()); if (waveformCleanup) waveformCleanup(); stopWaveform(waveformCanvas); await processAudio(audioBlob); }; mediaRecorder.start(); isRecording = true; micBtn.classList.add('recording'); micStatus.textContent = "Recording... Tap to stop"; // Start waveform visualization waveformCleanup = startWaveform(waveformCanvas, stream); } catch (err) { console.error("Microphone access denied:", err); alert("Microphone access is required to record symptoms."); } } else { mediaRecorder.stop(); isRecording = false; micBtn.classList.remove('recording'); micStatus.textContent = "Processing..."; } }); async function processAudio(blob) { try { // Convert blob to base64 const reader = new FileReader(); reader.readAsDataURL(blob); reader.onloadend = async () => { let base64data = reader.result; // Remove data URL prefix (e.g., "data:audio/webm;base64,") base64data = base64data.split(',')[1]; micStatus.innerHTML = ' Transcribing...'; const result = await gradioClient.predict("/transcribe", { audio_b64: base64data }); if (result.data[0] && result.data[0].text) { state.symptomsText = result.data[0].text; transcriptText.textContent = state.symptomsText; transcriptContainer.classList.add('visible'); micStatus.textContent = "Audio transcribed."; checkReadyForTriage(); } else if (result.data[0] && result.data[0].error) { throw new Error(result.data[0].error); } else { throw new Error("No transcription returned"); } }; } catch (err) { console.error("Transcription failed:", err); micStatus.textContent = "Error transcribing audio."; } } // Editable transcript editTranscriptBtn.addEventListener('click', () => { transcriptText.contentEditable = "true"; transcriptText.focus(); }); transcriptText.addEventListener('blur', () => { transcriptText.contentEditable = "false"; state.symptomsText = transcriptText.textContent.trim(); }); // --- Camera & Photo Flow --- openCameraBtn.addEventListener('click', async () => { try { currentCameraStream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: 'environment' } }); cameraPreview.srcObject = currentCameraStream; cameraPreview.classList.add('visible'); cameraPlaceholder.style.display = 'none'; photoPreview.classList.remove('visible'); captureBtn.style.display = 'block'; openCameraBtn.style.display = 'none'; uploadPhotoBtn.style.display = 'none'; } catch (err) { console.error("Camera access denied:", err); alert("Camera access is required. Try uploading a photo instead."); } }); captureBtn.addEventListener('click', () => { if (!currentCameraStream) return; const canvas = document.createElement('canvas'); canvas.width = cameraPreview.videoWidth; canvas.height = cameraPreview.videoHeight; canvas.getContext('2d').drawImage(cameraPreview, 0, 0); // Convert canvas to blob (jpeg) canvas.toBlob(blob => { const file = new File([blob], "capture.jpg", { type: "image/jpeg" }); displayPhotoAndProcess(file); }, 'image/jpeg', 0.8); // Stop camera currentCameraStream.getTracks().forEach(track => track.stop()); currentCameraStream = null; cameraPreview.classList.remove('visible'); captureBtn.style.display = 'none'; openCameraBtn.style.display = 'block'; uploadPhotoBtn.style.display = 'block'; }); uploadPhotoBtn.addEventListener('click', () => { fileInput.click(); }); fileInput.addEventListener('change', (e) => { const file = e.target.files[0]; if (file) displayPhotoAndProcess(file); }); // Drag and drop onto placeholder cameraPlaceholder.addEventListener('dragover', (e) => { e.preventDefault(); cameraPlaceholder.style.borderColor = 'var(--green-primary)'; }); cameraPlaceholder.addEventListener('dragleave', (e) => { e.preventDefault(); cameraPlaceholder.style.borderColor = 'var(--border-color)'; }); cameraPlaceholder.addEventListener('drop', (e) => { e.preventDefault(); cameraPlaceholder.style.borderColor = 'var(--border-color)'; const file = e.dataTransfer.files[0]; if (file && file.type.startsWith('image/')) { displayPhotoAndProcess(file); } }); cameraPlaceholder.addEventListener('click', () => fileInput.click()); function displayPhotoAndProcess(file) { const imgUrl = URL.createObjectURL(file); photoPreview.src = imgUrl; photoPreview.classList.add('visible'); cameraPlaceholder.style.display = 'none'; // Convert to base64 and send const reader = new FileReader(); reader.readAsDataURL(file); reader.onloadend = async () => { let base64data = reader.result.split(',')[1]; photoDescriptionBox.innerHTML = ' Analyzing image...'; photoDescriptionBox.classList.add('visible'); try { const result = await gradioClient.predict("/describe_image", { image_b64: base64data }); if (result.data[0] && result.data[0].description) { state.visualDescription = result.data[0].description; photoDescriptionBox.innerHTML = `Vision findings: ${state.visualDescription}`; checkReadyForTriage(); } else { throw new Error(result.data[0]?.error || "No description returned"); } } catch (err) { console.error("Vision failed:", err); photoDescriptionBox.textContent = "Error analyzing image."; } }; } // --- Triage Flow --- function checkReadyForTriage() { if (state.symptomsText || state.visualDescription) { triageBtn.disabled = false; } } triageBtn.addEventListener('click', async () => { if (!gradioClient) return; triageBtn.disabled = true; const originalBtnHtml = triageBtn.innerHTML; triageBtn.innerHTML = ' Evaluating...'; // Show output section immediately to display loading state outputSection.classList.add('visible'); triageCardContainer.innerHTML = `
`; try { const result = await gradioClient.predict("/run_triage", { symptoms_text: state.symptomsText, visual_description: state.visualDescription }); const report = result.data[0]; if (report.error) throw new Error(report.error); // Render Card renderTriageCard(triageCardContainer, report); // Build summary texts for TTS const hindiText = buildHindiSummary(report); const englishText = buildEnglishSummary(report); // Request audio asynchronously (don't block the UI) requestTTS(hindiText, "hindi"); requestTTS(englishText, "english"); } catch (err) { console.error("Triage failed:", err); triageCardContainer.innerHTML = `

❌ Error generating triage report: ${err.message}

`; } finally { triageBtn.innerHTML = originalBtnHtml; triageBtn.disabled = false; } }); // --- TTS & Audio Player --- function buildHindiSummary(report) { let actions = (report.immediate_actions || []).join("। "); let referText = report.refer_to_doctor ? "डॉक्टर से संपर्क करने की सलाह दी जाती है।" : "डॉक्टर से तुरंत परामर्श की आवश्यकता नहीं है।"; let reasonText = report.refer_reason ? \` रेफरल का कारण: \${report.refer_reason}।\` : ""; return \`स्वास्थ्य मूल्यांकन रिपोर्ट। संभावित स्थिति: \${report.likely_condition}। गंभीरता स्तर: पांच में से \${report.severity}। आवश्यक कदम: \${actions}। \${referText}\${reasonText}\`; } function buildEnglishSummary(report) { let actions = (report.immediate_actions || []).join(". "); let referText = report.refer_to_doctor ? "Referral recommended." : "Consultation not immediately required."; let reasonText = report.refer_reason ? \` Reason: \${report.refer_reason}.\` : ""; return \`Triage complete. Likely condition: \${report.likely_condition}. Severity: \${report.severity} out of 5. Actions: \${actions}. Doctor referral status: \${referText}\${reasonText}\`; } async function requestTTS(text, language) { try { const result = await gradioClient.predict("/synthesize_audio", { text, language }); const data = result.data[0]; if (data.audio_b64) { const dataUrl = \`data:\${data.mime_type};base64,\${data.audio_b64}\`; setupCustomAudioPlayer(\`audio-\${language}\`, dataUrl); } } catch (err) { console.error(\`\${language} TTS failed:\`, err); } } function setupCustomAudioPlayer(containerId, dataUrl) { const container = document.getElementById(containerId); if (!container) return; const audioEl = container.querySelector('audio'); const playBtn = container.querySelector('.play-btn'); const playIcon = playBtn.querySelector('.play-icon'); const pauseIcon = playBtn.querySelector('.pause-icon'); const progressFill = container.querySelector('.progress-fill'); const progressBar = container.querySelector('.progress-bar'); const timeDisplay = container.querySelector('.time-display'); audioEl.src = dataUrl; // Play/Pause playBtn.onclick = () => { if (audioEl.paused) { audioEl.play(); playIcon.style.display = 'none'; pauseIcon.style.display = 'block'; } else { audioEl.pause(); playIcon.style.display = 'block'; pauseIcon.style.display = 'none'; } }; // Update Progress audioEl.ontimeupdate = () => { const percent = (audioEl.currentTime / audioEl.duration) * 100; if (!isNaN(percent)) { progressFill.style.width = \`\${percent}%\`; } // Format time display (e.g. 0:05) const current = Math.floor(audioEl.currentTime); const mins = Math.floor(current / 60); const secs = current % 60; timeDisplay.textContent = \`\${mins}:\${secs.toString().padStart(2, '0')}\`; }; // End event audioEl.onended = () => { playIcon.style.display = 'block'; pauseIcon.style.display = 'none'; progressFill.style.width = '0%'; timeDisplay.textContent = '0:00'; }; // Click to seek progressBar.onclick = (e) => { const rect = progressBar.getBoundingClientRect(); const pos = (e.clientX - rect.left) / rect.width; if (!isNaN(audioEl.duration)) { audioEl.currentTime = pos * audioEl.duration; } }; } // --- Reset Flow --- resetBtn.addEventListener('click', () => { state = { symptomsText: "", visualDescription: "", hindiAudioB64: "", englishAudioB64: "" }; // Reset Mic UI transcriptContainer.classList.remove('visible'); transcriptText.textContent = ""; micStatus.textContent = "Tap to record"; stopWaveform(waveformCanvas); // Reset Camera UI photoPreview.src = ""; photoPreview.classList.remove('visible'); cameraPreview.classList.remove('visible'); cameraPlaceholder.style.display = 'flex'; openCameraBtn.style.display = 'block'; uploadPhotoBtn.style.display = 'block'; captureBtn.style.display = 'none'; photoDescriptionBox.classList.remove('visible'); photoDescriptionBox.textContent = ""; fileInput.value = ""; // Reset Output triageBtn.disabled = true; outputSection.classList.remove('visible'); triageCardContainer.innerHTML = ""; // Stop any playing audio document.querySelectorAll('audio').forEach(a => { a.pause(); a.src = ""; }); window.scrollTo({ top: 0, behavior: 'smooth' }); }); // Draw initial flat waveform line document.addEventListener('DOMContentLoaded', () => { stopWaveform(waveformCanvas); });