/** * FADA Mobile - Main application logic. * Manages UI state, chat interaction, and autonomous pipeline. */ import { loadModel, runInference, getLoadingStatus, getActiveDevice, testTextOnly } from "./model.js"; import { parseModelOutput, rescalePredictions } from "./output-parser.js"; import { parseIntent } from "./intent-parser.js"; import { parseInterpretationJson, extractClsLabel, mapInterpretationToClasses } from "./class-mapper.js"; import { drawAnnotations } from "./visualizer.js"; import { INTERPRET_PROMPT, CLASSIFY_PROMPT, DEFAULT_DETECT_CLASSES, MAX_NEW_TOKENS, TEMPERATURE, } from "./constants.js"; // State let currentImage = null; let imageFile = null; let interpretation = null; let classification = null; let detClasses = null; let segClasses = null; let isProcessing = false; // DOM refs const chatLog = document.getElementById("chat-log"); const userInput = document.getElementById("user-input"); const sendBtn = document.getElementById("send-btn"); const imageInput = document.getElementById("image-input"); const imagePreview = document.getElementById("image-preview"); const autoBtn = document.getElementById("auto-btn"); const loadBtn = document.getElementById("load-btn"); const deviceSelect = document.getElementById("device-select"); const statusEl = document.getElementById("status"); const progressBar = document.getElementById("progress-bar"); const progressText = document.getElementById("progress-text"); // --------------------------------------------------------------------------- // Init // --------------------------------------------------------------------------- sendBtn.addEventListener("click", handleSend); userInput.addEventListener("keydown", (e) => { if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); handleSend(); } }); imageInput.addEventListener("change", handleImageUpload); autoBtn.addEventListener("click", handleAutonomous); loadBtn.addEventListener("click", handleLoadModel); checkWebGPU(); async function checkWebGPU() { if (!navigator.gpu) { addMessage("system", "WebGPU NOT available. Select CPU (WASM) to proceed, or use Chrome/Edge 113+."); deviceSelect.value = "wasm"; updateGpuInfo("No WebGPU"); return; } try { // Request high-performance adapter to prefer NVIDIA discrete GPU over Intel integrated const adapter = await navigator.gpu.requestAdapter({ powerPreference: "high-performance", }); if (adapter) { const info = await adapter.requestAdapterInfo?.() || {}; const desc = info.description || info.device || "unknown GPU"; const vendor = info.vendor || ""; const fullDesc = `${vendor} ${desc}`.trim(); addMessage("system", `WebGPU available (high-performance): ${fullDesc}`); updateGpuInfo(fullDesc); console.log("[FADA] WebGPU adapter info:", JSON.stringify(info)); } else { addMessage("system", "WebGPU adapter not found. Defaulting to CPU."); deviceSelect.value = "wasm"; updateGpuInfo("No adapter"); } } catch (e) { console.warn("[FADA] WebGPU check error:", e); } } function updateGpuInfo(text) { const el = document.getElementById("gpu-info"); if (el) el.textContent = `GPU: ${text}`; } // --------------------------------------------------------------------------- // Model loading // --------------------------------------------------------------------------- async function handleLoadModel() { if (getLoadingStatus() === "ready") { setStatus("Model already loaded"); return; } const device = deviceSelect.value; loadBtn.disabled = true; deviceSelect.disabled = true; setStatus(`Loading model on ${device}...`); showProgress(true); let shaderTimer = null; let shaderStart = 0; try { shaderStart = Date.now(); await loadModel((info) => { if (info.status === "downloading") { const pct = info.progress ? info.progress.toFixed(1) : "0"; const file = info.file ? info.file.split("/").pop() : ""; setProgress(info.progress || 0, `Downloading ${file} (${pct}%)`); } else if (info.status === "compiling" || info.note?.includes("compiling")) { setStatus(info.note || "Compiling GPU shaders..."); setProgress(100, "Download complete - compiling shaders..."); // Start animated compilation timer if (!shaderTimer) { shaderTimer = setInterval(() => { const elapsed = Math.floor((Date.now() - shaderStart) / 1000); const dots = '.'.repeat((elapsed % 3) + 1); setStatus(`Compiling GPU shaders${dots} (${elapsed}s elapsed, typically 2-5 min)`); }, 1000); } } else if (info.status === "loading") { setProgress(0, `Loading ${info.file}...`); } else if (info.status === "ready") { setProgress(100, "Model ready!"); } }, device); if (shaderTimer) clearInterval(shaderTimer); setStatus(`Model loaded on ${device} - ready for inference`); showProgress(false); enableChat(true); document.getElementById('test-text-btn').disabled = false; if (device === "webgpu") { addMessage("system", "Model loaded on GPU. First inference may take 1-2 min for shader compilation."); } else { addMessage("system", "Model loaded on CPU. Inference will be slower but reliable."); } } catch (err) { if (shaderTimer) clearInterval(shaderTimer); setStatus(`Error loading model: ${err.message}`); showProgress(false); loadBtn.disabled = false; deviceSelect.disabled = false; addMessage("system", `Failed to load model: ${err.message}`); } } function enableChat(enabled) { sendBtn.disabled = !enabled; userInput.disabled = !enabled; autoBtn.disabled = !enabled; if (enabled) userInput.focus(); } // --------------------------------------------------------------------------- // Image handling // --------------------------------------------------------------------------- function handleImageUpload(e) { const file = e.target.files[0]; if (!file) return; imageFile = file; const img = new Image(); img.onload = () => { const maxSide = 1024; let w = img.width, h = img.height; if (Math.max(w, h) > maxSide) { const scale = maxSide / Math.max(w, h); w = Math.round(w * scale); h = Math.round(h * scale); } const canvas = document.createElement("canvas"); canvas.width = w; canvas.height = h; canvas.getContext("2d").drawImage(img, 0, 0, w, h); const resized = new Image(); resized.onload = () => { currentImage = resized; imagePreview.innerHTML = ""; imagePreview.appendChild(resized.cloneNode()); interpretation = null; classification = null; detClasses = null; segClasses = null; addMessage("system", `Image uploaded (${w}x${h}). Type "interpret", "detect", or "segment" in the chat.`); }; resized.src = canvas.toDataURL(); }; img.src = URL.createObjectURL(file); } // --------------------------------------------------------------------------- // Chat handling // --------------------------------------------------------------------------- async function handleSend() { const text = userInput.value.trim(); if (!text && !currentImage) return; if (isProcessing) return; if (getLoadingStatus() !== "ready") { addMessage("system", "Please load the model first (click 'Load Model')."); return; } addMessage("user", text || "(image uploaded)"); userInput.value = ""; isProcessing = true; sendBtn.disabled = true; try { await processChatTurn(text); } catch (err) { console.error("Inference error:", err); removeThinking(); addMessage("system", `Error: ${err.message}`); } isProcessing = false; sendBtn.disabled = false; userInput.focus(); } async function processChatTurn(userText) { const intent = parseIntent(userText, !!currentImage, !!interpretation); if (intent.task === "greeting") { addMessage("assistant", "Welcome to FADA Mobile! Upload a fetal ultrasound image and I'll provide clinical interpretation. You can then ask me to detect or segment specific anatomical structures."); return; } if (intent.task === "general") { addMessage("assistant", "I can help with fetal ultrasound analysis. Try:\n- Upload an image and ask me to **interpret** it\n- Ask me to **detect** specific structures (e.g., 'detect brain')\n- Ask me to **segment** anatomy (e.g., 'segment cardiac')"); return; } if (!currentImage) { addMessage("assistant", "Please upload a fetal ultrasound image first."); return; } if (intent.task === "interpret") { addMessage("assistant", "Analyzing image...", "thinking"); setStatus("Running interpretation..."); const t0 = performance.now(); let tokensSoFar = 0; const rawInterp = await runInference(currentImage, INTERPRET_PROMPT, { temperature: TEMPERATURE, onToken: (tok, n) => { tokensSoFar = n; setStatus(`Interpreting... (${n} tokens generated)`); }, }); const interpTime = ((performance.now() - t0) / 1000).toFixed(1); interpretation = parseInterpretationJson(rawInterp); setStatus("Running classification..."); const rawCls = await runInference(currentImage, CLASSIFY_PROMPT, { maxNewTokens: 256, temperature: TEMPERATURE, }); const clsResult = parseInterpretationJson(rawCls); classification = extractClsLabel(clsResult); if (interpretation._parseSuccess) { const mapped = mapInterpretationToClasses(interpretation, classification); detClasses = mapped.det; segClasses = mapped.seg; } removeThinking(); let response = formatInterpretation(interpretation); if (classification) response += `\n**Classification:** ${classification}`; response += `\n\n*${tokensSoFar} tokens in ${interpTime}s on ${getActiveDevice()}*`; response += "\n\n---\nYou can now ask me to **detect** or **segment** specific structures."; addMessage("assistant", response); setStatus("Ready"); return; } if (intent.task === "classify") { addMessage("assistant", "Classifying...", "thinking"); setStatus("Running classification..."); const rawCls = await runInference(currentImage, CLASSIFY_PROMPT, { maxNewTokens: 256, temperature: TEMPERATURE }); const clsResult = parseInterpretationJson(rawCls); classification = extractClsLabel(clsResult); removeThinking(); addMessage("assistant", `**Classification:** ${classification || "Unknown"}`); setStatus("Ready"); return; } if (intent.task === "detect") { const classes = intent.classes || detClasses || DEFAULT_DETECT_CLASSES; const prompt = buildDetectPrompt(classes); addMessage("assistant", `Detecting: ${classes}...`, "thinking"); setStatus("Running detection..."); const raw = await runInference(currentImage, prompt, { temperature: TEMPERATURE, onToken: (tok, n) => setStatus(`Detecting... (${n} tokens)`), }); removeThinking(); showAnnotation(raw, "Detection"); setStatus("Ready"); return; } if (intent.task === "segment") { const classes = intent.segClasses || intent.classes || segClasses || detClasses || DEFAULT_DETECT_CLASSES; const prompt = buildSegmentPrompt(classes); addMessage("assistant", `Segmenting: ${classes}...`, "thinking"); setStatus("Running segmentation..."); const raw = await runInference(currentImage, prompt, { temperature: TEMPERATURE, onToken: (tok, n) => setStatus(`Segmenting... (${n} tokens)`), }); removeThinking(); showAnnotation(raw, "Segmentation"); setStatus("Ready"); return; } } // --------------------------------------------------------------------------- // Autonomous pipeline // --------------------------------------------------------------------------- async function handleAutonomous() { if (!currentImage) { addMessage("system", "Please upload an image first."); return; } if (getLoadingStatus() !== "ready") { addMessage("system", "Please load the model first."); return; } if (isProcessing) return; isProcessing = true; autoBtn.disabled = true; sendBtn.disabled = true; try { addMessage("system", "Starting autonomous 5-phase pipeline..."); // Phase 1: Interpret setStatus("Phase 1/5: Interpreting..."); addMessage("assistant", "Phase 1: Interpreting image...", "thinking"); const t0 = performance.now(); const rawInterp = await runInference(currentImage, INTERPRET_PROMPT, { temperature: TEMPERATURE, onToken: (tok, n) => setStatus(`Phase 1/5: Interpreting... (${n} tokens)`), }); const interpTime = ((performance.now() - t0) / 1000).toFixed(1); interpretation = parseInterpretationJson(rawInterp); removeThinking(); addMessage("assistant", formatInterpretation(interpretation) + `\n*${interpTime}s*`); // Phase 2: Classify setStatus("Phase 2/5: Classifying..."); const rawCls = await runInference(currentImage, CLASSIFY_PROMPT, { maxNewTokens: 256, temperature: TEMPERATURE }); const clsResult = parseInterpretationJson(rawCls); classification = extractClsLabel(clsResult); addMessage("assistant", `**Classification:** ${classification || "Unknown"}`); // Phase 3: Map setStatus("Phase 3/5: Mapping classes..."); let mapped; if (interpretation._parseSuccess) { mapped = mapInterpretationToClasses(interpretation, classification); } else { mapped = { det: DEFAULT_DETECT_CLASSES, seg: null, tier: "fallback" }; } detClasses = mapped.det; segClasses = mapped.seg; addMessage("system", `Mapped: detect=[${mapped.det}], segment=[${mapped.seg || "none"}], tier=${mapped.tier}`); // Phase 4: Detect setStatus("Phase 4/5: Detecting..."); addMessage("assistant", `Detecting: ${detClasses}...`, "thinking"); const rawDet = await runInference(currentImage, buildDetectPrompt(detClasses), { temperature: TEMPERATURE, onToken: (tok, n) => setStatus(`Phase 4/5: Detecting... (${n} tokens)`), }); removeThinking(); showAnnotation(rawDet, "Detection"); // Phase 5: Segment (conditional) if (segClasses) { setStatus("Phase 5/5: Segmenting..."); addMessage("assistant", `Segmenting: ${segClasses}...`, "thinking"); const rawSeg = await runInference(currentImage, buildSegmentPrompt(segClasses), { temperature: TEMPERATURE, onToken: (tok, n) => setStatus(`Phase 5/5: Segmenting... (${n} tokens)`), }); removeThinking(); showAnnotation(rawSeg, "Segmentation"); } else { addMessage("system", "Phase 5: Segmentation skipped (no classes mapped)."); } setStatus("Pipeline complete!"); addMessage("system", "Autonomous pipeline complete. You can ask follow-up questions."); } catch (err) { removeThinking(); console.error("Pipeline error:", err); addMessage("system", `Pipeline error: ${err.message}`); setStatus("Error"); } isProcessing = false; autoBtn.disabled = false; sendBtn.disabled = false; } // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- function buildDetectPrompt(classes) { return `Detect all instances of the following anatomical structures in this fetal ultrasound image: ${classes}. For each detection, return a JSON object with "bbox_2d" as [x_min, y_min, x_max, y_max] in normalized 0-1000 coordinates and "label" as the structure name.`; } function buildSegmentPrompt(classes) { return `Detect and segment all instances of the following anatomical structures in this fetal ultrasound image: ${classes}. For each instance, return a JSON object with "bbox_2d" as [x_min, y_min, x_max, y_max] in normalized 0-1000 coordinates, "label" as the structure name, and "mask" as a list of [x, y] polygon vertices in normalized 0-1000 coordinates.`; } function formatInterpretation(interp) { if (!interp._parseSuccess) { return `**Interpretation (raw):**\n\`\`\`\n${(interp._rawText || "").slice(0, 1000)}\n\`\`\``; } const fields = [ ["Anatomical Structures", "anatomical_structures"], ["Fetal Orientation", "fetal_orientation"], ["Imaging Plane", "imaging_plane"], ["Biometric Measurements", "biometric_measurements"], ["Gestational Age", "gestational_age"], ["Image Quality", "image_quality"], ["Normality Assessment", "normality_assessment"], ["Clinical Recommendations", "clinical_recommendations"], ]; let parts = "**Clinical Interpretation:**\n\n"; for (const [title, key] of fields) { let value = interp[key] || "N/A"; if (typeof value === "object") value = JSON.stringify(value, null, 2); parts += `**${title}:** ${value}\n\n`; } return parts; } function showAnnotation(rawText, taskName) { try { const parsed = parseModelOutput(rawText); const rescaled = rescalePredictions(parsed, currentImage.width, currentImage.height); const nDet = rescaled.detections.length; const nSeg = rescaled.segmentations.length; const nKp = rescaled.keypoints.length; if (nDet + nSeg + nKp === 0) { addMessage("assistant", `**${taskName}:** No structures found for the requested classes.`); return; } const canvas = document.createElement("canvas"); drawAnnotations(canvas, currentImage, rescaled); const imgEl = new Image(); imgEl.src = canvas.toDataURL(); imgEl.className = "annotation-img"; const labels = new Set(); rescaled.detections.forEach(d => labels.add(d.label)); rescaled.segmentations.forEach(s => labels.add(s.label)); rescaled.keypoints.forEach(k => labels.add(k.label)); const parts = []; if (nDet > 0) parts.push(`${nDet} detections`); if (nSeg > 0) parts.push(`${nSeg} masks`); if (nKp > 0) parts.push(`${nKp} keypoints`); addMessage("assistant", `**${taskName} results** (${parts.join(", ")}: ${[...labels].sort().join(", ")})`); chatLog.lastElementChild.appendChild(imgEl); } catch (err) { addMessage("assistant", `**${taskName}** completed but annotation failed. Raw output:\n\`\`\`\n${rawText.slice(0, 500)}\n\`\`\``); } } function addMessage(role, text, extraClass = "") { const div = document.createElement("div"); div.className = `message ${role} ${extraClass}`.trim(); let html = text .replace(/\*\*(.*?)\*\*/g, "$1") .replace(/\n/g, "
"); div.innerHTML = html; chatLog.appendChild(div); chatLog.scrollTop = chatLog.scrollHeight; } function removeThinking() { const el = chatLog.querySelector(".thinking"); if (el) el.remove(); } function setStatus(text) { statusEl.textContent = text; } function showProgress(show) { document.getElementById("progress-container").style.display = show ? "block" : "none"; } function setProgress(pct, text) { progressBar.style.width = `${pct}%`; progressText.textContent = text || ""; } // --------------------------------------------------------------------------- // Text-only diagnostic test // --------------------------------------------------------------------------- window.runTextTest = async () => { const btn = document.getElementById('test-text-btn'); btn.disabled = true; btn.textContent = "Testing..."; try { const result = await testTextOnly(); // Display result in chat addMessage("assistant", `Diagnostic: ${result}`); alert(result); } catch(e) { alert("Test failed: " + e.message); console.error("[FADA-TEST] Error:", e); addMessage("system", `Text-only test error: ${e.message}`); } finally { btn.disabled = false; btn.textContent = "Test Text-Only"; } };