// ========================================================================== // MEMRL CANVAS FRONTEND LOGIC (ADVANCED V2) // ========================================================================== // Drawing State let currentTool = 'circle'; let currentColor = 'red'; let strokeSize = 100; let shapes = []; let undoStack = []; // Drag and drop state let draggedShape = null; let draggedIndex = -1; let dragOffsetX = 0; let dragOffsetY = 0; // Audio recording state let mediaRecorder = null; let audioChunks = []; let audioContext = null; let analyser = null; let animationFrameId = null; let isRecording = false; // MemRL state variables let lastTranscription = ""; let lastInterpretedAction = []; // Correction mode state (better preview-before-reinforce flow for MemRL) let correctionSnapshot = null; let correctionControls = null; // Tracing animation state let animationProgress = 1.0; let animationActive = false; // ========================================== // 1. Coordinate Mapping & Shape Drawing Math // ========================================== const canvas = document.getElementById('drawingCanvas'); const ctx = canvas.getContext('2d'); function resolveX(val) { if (val === 'left') return 120; if (val === 'right') return 380; if (val === 'center') return 250; if (typeof val === 'number') return val; const num = parseFloat(val); return isNaN(num) ? 250 : num; } function resolveY(val) { if (val === 'top') return 120; if (val === 'bottom') return 380; if (val === 'center') return 250; if (typeof val === 'number') return val; const num = parseFloat(val); return isNaN(num) ? 250 : num; } // Draw a blueprint engineering grid background function drawGrid() { if (!document.getElementById('gridToggle').checked) return; ctx.strokeStyle = '#e2e8f0'; ctx.lineWidth = 1; // Vertical grid lines for (let x = 25; x < canvas.width; x += 25) { ctx.beginPath(); ctx.moveTo(x, 0); ctx.lineTo(x, canvas.height); ctx.strokeStyle = (x % 100 === 0) ? '#cbd5e1' : '#f1f5f9'; ctx.stroke(); } // Horizontal grid lines for (let y = 25; y < canvas.height; y += 25) { ctx.beginPath(); ctx.moveTo(0, y); ctx.lineTo(canvas.width, y); ctx.strokeStyle = (y % 100 === 0) ? '#cbd5e1' : '#f1f5f9'; ctx.stroke(); } } // Math equations for drawing academic shapes function drawCircle(ctx, cx, cy, size, drawStroke) { ctx.beginPath(); ctx.arc(cx, cy, size / 2, 0, Math.PI * 2); if (drawStroke) ctx.stroke(); else ctx.fill(); } function drawSquare(ctx, cx, cy, size, drawStroke) { ctx.beginPath(); ctx.rect(cx - size / 2, cy - size / 2, size, size); if (drawStroke) ctx.stroke(); else ctx.fill(); } function drawRectangle(ctx, cx, cy, size, drawStroke) { ctx.beginPath(); ctx.rect(cx - size, cy - size / 2, size * 2, size); if (drawStroke) ctx.stroke(); else ctx.fill(); } function drawTriangle(ctx, cx, cy, size, drawStroke) { ctx.beginPath(); ctx.moveTo(cx, cy - size / 2); ctx.lineTo(cx - size / 2, cy + size / 2); ctx.lineTo(cx + size / 2, cy + size / 2); ctx.closePath(); if (drawStroke) ctx.stroke(); else ctx.fill(); } function drawPentagon(ctx, cx, cy, size, drawStroke) { ctx.beginPath(); const r = size / 2; for (let i = 0; i < 5; i++) { const angle = (i * 2 * Math.PI / 5) - Math.PI / 2; const x = cx + r * Math.cos(angle); const y = cy + r * Math.sin(angle); if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y); } ctx.closePath(); if (drawStroke) ctx.stroke(); else ctx.fill(); } function drawHexagon(ctx, cx, cy, size, drawStroke) { ctx.beginPath(); const r = size / 2; for (let i = 0; i < 6; i++) { const angle = (i * 2 * Math.PI / 6) - Math.PI / 2; const x = cx + r * Math.cos(angle); const y = cy + r * Math.sin(angle); if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y); } ctx.closePath(); if (drawStroke) ctx.stroke(); else ctx.fill(); } function drawStar(ctx, cx, cy, size, drawStroke) { ctx.beginPath(); const rOuter = size / 2; const rInner = size / 4.5; const points = 5; for (let i = 0; i < 2 * points; i++) { const angle = (i * Math.PI / points) - Math.PI / 2; const r = (i % 2 === 0) ? rOuter : rInner; const x = cx + r * Math.cos(angle); const y = cy + r * Math.sin(angle); if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y); } ctx.closePath(); if (drawStroke) ctx.stroke(); else ctx.fill(); } function drawOval(ctx, cx, cy, size, drawStroke) { ctx.beginPath(); ctx.ellipse(cx, cy, size / 2, size / 3, 0, 0, Math.PI * 2); if (drawStroke) ctx.stroke(); else ctx.fill(); } function drawHeart(ctx, cx, cy, size, drawStroke) { ctx.beginPath(); const d = size * 0.72; const x = cx; const y = cy - d / 4; ctx.moveTo(x, y); ctx.bezierCurveTo(x - d / 2, y - d / 2, x - d, y + d / 6, x, y + d * 0.7); ctx.bezierCurveTo(x + d, y + d / 6, x + d / 2, y - d / 2, x, y); ctx.closePath(); if (drawStroke) ctx.stroke(); else ctx.fill(); } function drawTextShape(ctx, shape, progress = 1.0) { const tx = resolveX(shape.x !== undefined ? shape.x : 'center'); const ty = resolveY(shape.y !== undefined ? shape.y : 'center'); const fsize = shape.size || 48; const label = shape.text || 'text'; ctx.save(); ctx.fillStyle = shape.color || 'black'; ctx.font = `${fsize}px sans-serif`; ctx.textAlign = 'center'; ctx.textBaseline = 'middle'; // Simple fade-in for animation consistency ctx.globalAlpha = Math.min(1.0, Math.max(0.2, progress)); ctx.fillText(label, tx, ty); ctx.restore(); } function drawLineShape(ctx, startX, startY, endX, endY, progress = 1.0) { ctx.beginPath(); ctx.moveTo(startX, startY); // Draw progressive line length based on animation progress const curX = startX + (endX - startX) * progress; const curY = startY + (endY - startY) * progress; ctx.lineTo(curX, curY); ctx.stroke(); } function drawArrowShape(ctx, startX, startY, endX, endY, progress = 1.0) { ctx.beginPath(); ctx.moveTo(startX, startY); const curX = startX + (endX - startX) * progress; const curY = startY + (endY - startY) * progress; ctx.lineTo(curX, curY); ctx.stroke(); // Draw arrowhead only at the end if (progress >= 0.9) { const angle = Math.atan2(endY - startY, endX - startX); const headLength = 14; ctx.beginPath(); ctx.moveTo(endX, endY); ctx.lineTo(endX - headLength * Math.cos(angle - Math.PI / 6), endY - headLength * Math.sin(angle - Math.PI / 6)); ctx.lineTo(endX - headLength * Math.cos(angle + Math.PI / 6), endY - headLength * Math.sin(angle + Math.PI / 6)); ctx.closePath(); ctx.fill(); } } // Path / Freehand Operations Runner function drawPathShape(ctx, shape, drawStroke, progress = 1.0) { const ops = shape.operations || []; if (ops.length === 0) return; const opsToDraw = Math.floor(ops.length * progress); let px = resolveX('center'); let py = resolveY('center'); ctx.beginPath(); ops.forEach((op, index) => { if (index > opsToDraw) return; if (op.type === 'start') { px = resolveX(op.x); py = resolveY(op.y); ctx.moveTo(px, py); } else if (op.type === 'line') { px += op.dx || 0; py += op.dy || 0; ctx.lineTo(px, py); } else if (op.type === 'move') { px += op.dx || 0; py += op.dy || 0; ctx.moveTo(px, py); } else if (op.type === 'dot') { const r = op.radius || 6; ctx.arc(px, py, r, 0, Math.PI * 2); ctx.fill(); ctx.beginPath(); ctx.moveTo(px, py); } }); if (opsToDraw > 0) { ctx.stroke(); } } // ========================================== // 2. Render Pipeline // ========================================== function renderAllShapes() { // Fill canvas background ctx.fillStyle = '#ffffff'; ctx.fillRect(0, 0, canvas.width, canvas.height); // Draw blueprint lines drawGrid(); // Draw shapes in order shapes.forEach((shape, index) => { const isLast = (index === shapes.length - 1); const progress = (isLast && animationActive) ? animationProgress : 1.0; ctx.strokeStyle = shape.color || 'black'; ctx.fillStyle = shape.color || 'black'; ctx.lineWidth = shape.thickness || 4; ctx.lineCap = 'round'; ctx.lineJoin = 'round'; const cx = resolveX(shape.x !== undefined ? shape.x : 'center'); const cy = resolveY(shape.y !== undefined ? shape.y : 'center'); const size = shape.size || 100; // 1. Path-based drawings if (shape.shape === 'path') { drawPathShape(ctx, shape, true, progress); return; } // 2. Lines & Arrows if (shape.shape === 'line') { const sx = resolveX(shape.start_x !== undefined ? shape.start_x : 'center'); const sy = resolveY(shape.start_y !== undefined ? shape.start_y : 'center'); const ex = resolveX(shape.end_x !== undefined ? shape.end_x : 350); const ey = resolveY(shape.end_y !== undefined ? shape.end_y : 350); drawLineShape(ctx, sx, sy, ex, ey, progress); return; } if (shape.shape === 'arrow') { const sx = resolveX(shape.start_x !== undefined ? shape.start_x : 'center'); const sy = resolveY(shape.start_y !== undefined ? shape.start_y : 'center'); const ex = resolveX(shape.end_x !== undefined ? shape.end_x : 350); const ey = resolveY(shape.end_y !== undefined ? shape.end_y : 350); drawArrowShape(ctx, sx, sy, ex, ey, progress); return; } // 3. Clear canvas command if (shape.shape === 'clear') { ctx.fillStyle = '#ffffff'; ctx.fillRect(0, 0, canvas.width, canvas.height); drawGrid(); return; } // 3.5 Text labels (Agents.md roadmap item) if (shape.shape === 'text') { const tprog = (isLast && animationActive) ? animationProgress : 1.0; drawTextShape(ctx, shape, tprog); return; } // 4. Standard Geometry Tracing Animation logic if (progress < 0.6) { // Step 1: Trace outlines only (line dash animated) ctx.save(); const strokeProgress = progress / 0.6; ctx.setLineDash([size * 3 * strokeProgress, size * 3]); drawGeometry(ctx, cx, cy, size, shape.shape, true); ctx.restore(); } else { // Step 2: Flood fill opacity ctx.save(); ctx.globalAlpha = (progress - 0.6) / 0.4; drawGeometry(ctx, cx, cy, size, shape.shape, false); ctx.restore(); // Draw stroke outer bounds ctx.lineWidth = 2; drawGeometry(ctx, cx, cy, size, shape.shape, true); } }); } function drawGeometry(ctx, cx, cy, size, shapeType, drawStroke) { if (shapeType === 'circle') drawCircle(ctx, cx, cy, size, drawStroke); else if (shapeType === 'square') drawSquare(ctx, cx, cy, size, drawStroke); else if (shapeType === 'rectangle') drawRectangle(ctx, cx, cy, size, drawStroke); else if (shapeType === 'triangle') drawTriangle(ctx, cx, cy, size, drawStroke); else if (shapeType === 'pentagon') drawPentagon(ctx, cx, cy, size, drawStroke); else if (shapeType === 'hexagon') drawHexagon(ctx, cx, cy, size, drawStroke); else if (shapeType === 'star') drawStar(ctx, cx, cy, size, drawStroke); else if (shapeType === 'oval') drawOval(ctx, cx, cy, size, drawStroke); else if (shapeType === 'heart') drawHeart(ctx, cx, cy, size, drawStroke); } function triggerTraceAnimation() { if (!document.getElementById('animateToggle').checked) { animationProgress = 1.0; animationActive = false; renderAllShapes(); return; } animationProgress = 0.0; animationActive = true; animateLoop(); } function animateLoop() { if (!animationActive) return; animationProgress += 0.04; if (animationProgress >= 1.0) { animationProgress = 1.0; animationActive = false; } renderAllShapes(); if (animationActive) { requestAnimationFrame(animateLoop); } } function executeCanvasActions(actionList) { if (!actionList || !Array.isArray(actionList)) return; undoStack.push(JSON.parse(JSON.stringify(shapes))); actionList.forEach(action => { if (action.shape === 'clear') { shapes = []; } else { shapes.push(action); } }); triggerTraceAnimation(); } // ========================================== // 3. Dynamic Shape Drag & Drop Hit Detection // ========================================== function isPointInShape(px, py, shape) { const cx = resolveX(shape.x !== undefined ? shape.x : 'center'); const cy = resolveY(shape.y !== undefined ? shape.y : 'center'); const size = shape.size || 100; if (shape.shape === 'circle') { const dist = Math.sqrt((px - cx) ** 2 + (py - cy) ** 2); return dist <= size / 2; } else if (shape.shape === 'square') { return px >= cx - size / 2 && px <= cx + size / 2 && py >= cy - size / 2 && py <= cy + size / 2; } else if (shape.shape === 'rectangle') { return px >= cx - size && px <= cx + size && py >= cy - size / 2 && py <= cy + size / 2; } else if (shape.shape === 'path') { // Simple bounding box for paths based on center coordinate return px >= cx - 50 && px <= cx + 50 && py >= cy - 50 && py <= cy + 50; } else if (['triangle', 'star', 'pentagon', 'hexagon', 'oval', 'heart'].includes(shape.shape)) { // Bounding box approximation return px >= cx - size / 2 && px <= cx + size / 2 && py >= cy - size / 2 && py <= cy + size / 2; } else if (shape.shape === 'text') { // Approximate text hit box (centered). Width ~ 0.5 * size per char, height ~ 0.8 * size. const label = shape.text || 'text'; const approxW = Math.max(30, label.length * (size * 0.45)); const approxH = size * 0.8; return px >= cx - approxW / 2 && px <= cx + approxW / 2 && py >= cy - approxH / 2 && py <= cy + approxH / 2; } else if (shape.shape === 'line' || shape.shape === 'arrow') { // Distance from point to line segment formula const sx = resolveX(shape.start_x !== undefined ? shape.start_x : 'center'); const sy = resolveY(shape.start_y !== undefined ? shape.start_y : 'center'); const ex = resolveX(shape.end_x !== undefined ? shape.end_x : 350); const ey = resolveY(shape.end_y !== undefined ? shape.end_y : 350); const A = px - sx; const B = py - sy; const C = ex - sx; const D = ey - sy; const dot = A * C + B * D; const lenSq = C * C + D * D; let param = -1; if (lenSq !== 0) param = dot / lenSq; let xx, yy; if (param < 0) { xx = sx; yy = sy; } else if (param > 1) { xx = ex; yy = ey; } else { xx = sx + param * C; yy = sy + param * D; } const dist = Math.sqrt((px - xx) ** 2 + (py - yy) ** 2); return dist <= 12; // hit tolerance of 12 pixels } return false; } canvas.addEventListener('mousedown', (e) => { const rect = canvas.getBoundingClientRect(); const mx = e.clientX - rect.left; const my = e.clientY - rect.top; for (let i = shapes.length - 1; i >= 0; i--) { const shape = shapes[i]; if (isPointInShape(mx, my, shape)) { draggedShape = shape; draggedIndex = i; // Set drag offsets const cx = resolveX(shape.x !== undefined ? shape.x : 'center'); const cy = resolveY(shape.y !== undefined ? shape.y : 'center'); dragOffsetX = mx - cx; dragOffsetY = my - cy; canvas.style.cursor = 'grabbing'; break; } } }); canvas.addEventListener('mousemove', (e) => { const rect = canvas.getBoundingClientRect(); const mx = e.clientX - rect.left; const my = e.clientY - rect.top; if (draggedShape) { // Dragging line requires moving start and endpoints relatively if (draggedShape.shape === 'line' || draggedShape.shape === 'arrow') { const sx = resolveX(draggedShape.start_x !== undefined ? draggedShape.start_x : 'center'); const sy = resolveY(draggedShape.start_y !== undefined ? draggedShape.start_y : 'center'); const ex = resolveX(draggedShape.end_x !== undefined ? draggedShape.end_x : 350); const ey = resolveY(draggedShape.end_y !== undefined ? draggedShape.end_y : 350); const dx = mx - (sx + ex) / 2; const dy = my - (sy + ey) / 2; draggedShape.start_x = sx + dx; draggedShape.start_y = sy + dy; draggedShape.end_x = ex + dx; draggedShape.end_y = ey + dy; } else { // Standard shape movement coordinates mapping draggedShape.x = mx - dragOffsetX; draggedShape.y = my - dragOffsetY; } renderAllShapes(); } else { // Hover cursor styling let hover = false; for (let i = shapes.length - 1; i >= 0; i--) { if (isPointInShape(mx, my, shapes[i])) { hover = true; break; } } canvas.style.cursor = hover ? 'grab' : 'crosshair'; } }); canvas.addEventListener('mouseup', () => { if (draggedShape) { draggedShape = null; canvas.style.cursor = 'grab'; // Sync manual parameter box with newly dragged coordinates jsonBox.value = JSON.stringify(shapes, null, 2); } }); // ========================================== // 4. Audio Controls & Recording // ========================================== const micBtn = document.getElementById('micBtn'); const waveContainer = document.getElementById('recordingWave'); const recordingStatus = document.getElementById('recordingStatus'); micBtn.addEventListener('click', toggleRecording); async function toggleRecording() { if (isRecording) { stopRecording(); } else { await startRecording(); } } async function startRecording() { audioChunks = []; try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); let options = { mimeType: 'audio/webm' }; if (!MediaRecorder.isTypeSupported('audio/webm')) { options = { mimeType: 'audio/ogg' }; if (!MediaRecorder.isTypeSupported('audio/ogg')) { options = { mimeType: 'audio/mp4' }; } } mediaRecorder = new MediaRecorder(stream, options); mediaRecorder.ondataavailable = (e) => { if (e.data.size > 0) audioChunks.push(e.data); }; mediaRecorder.onstop = sendAudioToBackend; setupVisualizer(stream); mediaRecorder.start(); isRecording = true; micBtn.classList.add('recording'); waveContainer.classList.add('active'); recordingStatus.textContent = "Recording... Click again to process."; } catch (err) { console.error("Audio device access error:", err); recordingStatus.textContent = "Error: Microphone access denied."; } } function stopRecording() { if (mediaRecorder && isRecording) { mediaRecorder.stop(); mediaRecorder.stream.getTracks().forEach(track => track.stop()); isRecording = false; micBtn.classList.remove('recording'); waveContainer.classList.remove('active'); recordingStatus.textContent = "Processing speech..."; if (animationFrameId) cancelAnimationFrame(animationFrameId); if (audioContext) audioContext.close(); } } function setupVisualizer(stream) { audioContext = new (window.AudioContext || window.webkitAudioContext)(); analyser = audioContext.createAnalyser(); const source = audioContext.createMediaStreamSource(stream); source.connect(analyser); analyser.fftSize = 32; const bufferLength = analyser.frequencyBinCount; const dataArray = new Uint8Array(bufferLength); const bars = document.querySelectorAll('.wave-bar'); function draw() { if (!isRecording) return; animationFrameId = requestAnimationFrame(draw); analyser.getByteFrequencyData(dataArray); bars.forEach((bar, index) => { const value = dataArray[index % bufferLength]; const height = Math.max(4, Math.min(40, (value / 255) * 45)); bar.style.height = `${height}px`; }); } draw(); } // ========================================== // 5. API Loop Operations // ========================================== const transcriptionBox = document.getElementById('transcriptionBox'); const statusBadge = document.getElementById('statusBadge'); const jsonBox = document.getElementById('jsonBox'); const rewardCard = document.getElementById('rewardCard'); async function sendAudioToBackend() { const audioBlob = new Blob(audioChunks, { type: mediaRecorder.mimeType }); const formData = new FormData(); formData.append('audio', audioBlob, 'voice_command.wav'); try { const response = await fetch('/api/transcribe', { method: 'POST', body: formData }); if (!response.ok) throw new Error("Transcription server error"); const result = await response.json(); processASRResult(result); } catch (err) { console.error("Transcribe API communication failed:", err); recordingStatus.textContent = "Error transcribing audio command."; } } function processASRResult(result) { recordingStatus.textContent = "Command resolved."; // Exit any pending correction session when a new voice command arrives resetCorrectionState(); lastTranscription = result.transcription; lastInterpretedAction = result.action; transcriptionBox.textContent = lastTranscription ? `"${lastTranscription}"` : "None"; if (!lastTranscription || !lastInterpretedAction || lastInterpretedAction.length === 0) { updateBadge("IDLE", "state-idle"); jsonBox.value = ""; rewardCard.classList.add('disabled'); return; } // Always keep jsonBox as pure actions array so correction/preview editing works jsonBox.value = JSON.stringify(lastInterpretedAction, null, 2); rewardCard.classList.remove('disabled'); // Extra safety for demo: if actions empty but we have canonical, synthesize on client too if ((!lastInterpretedAction || lastInterpretedAction.length === 0) && result.canonical) { const c = result.canonical; const slots = c.slots || {}; lastInterpretedAction = [{ shape: slots.shape || (c.intent === 'draw_shape' ? 'circle' : c.intent) || 'circle', color: slots.color || 'black', size: slots.size || 100, x: slots.x || 'center', y: slots.y || 'center' }]; if (c.intent === 'clear') { lastInterpretedAction = [{ shape: 'clear', color: 'white', size: 0 }]; } jsonBox.value = JSON.stringify(lastInterpretedAction, null, 2); } // Show canonical info in status for demo visibility (without polluting the editable JSON box) if (result.canonical) { const canonStr = JSON.stringify(result.canonical); // Append a small note to transcription area for now (demo only) const current = transcriptionBox.textContent || ''; if (!current.includes('Canonical:')) { transcriptionBox.textContent = current + ` | Canonical: ${canonStr}`; } } // Rich status for Hackathon demo (advertises multilingual + canonical) const status = result.status || ""; if (status.includes("MemRL Template") || status.includes("Auto-Executed")) { updateBadge("MEMRL TEMPLATE (lang-agnostic)", "state-matched"); executeCanvasActions(lastInterpretedAction); } else if (status.includes("Low Confidence") || status.includes("Suggestion")) { updateBadge("MEMRL SUGGESTION (multilingual)", "state-suggest"); showSuggestionBanner(lastTranscription, lastInterpretedAction); } else if (status.includes("Gemma + RAG") || status.includes("canonical")) { const langNote = result.language ? ` (${result.language})` : ""; updateBadge(`GEMMA+RAG (canonical${langNote})`, "state-miss"); executeCanvasActions(lastInterpretedAction); } else { updateBadge("INTERPRETED", "state-miss"); executeCanvasActions(lastInterpretedAction); } } function updateBadge(text, className) { statusBadge.textContent = text; statusBadge.className = `status-badge ${className}`; } // ========================================== // 6. Suggestion Overlay Banner // ========================================== const suggestionBanner = document.getElementById('suggestionBanner'); const bannerYes = document.getElementById('bannerYes'); const bannerNo = document.getElementById('bannerNo'); function showSuggestionBanner(query, actionList) { const desc = `${actionList.length} action shape(s) resolved.`; suggestionBanner.querySelector('.banner-text').textContent = `MemRL Suggestion: Did you mean to draw: "${query}"? (${desc})`; suggestionBanner.classList.remove('hidden'); } bannerYes.addEventListener('click', () => { suggestionBanner.classList.add('hidden'); executeCanvasActions(lastInterpretedAction); sendFeedback(1.0); }); bannerNo.addEventListener('click', () => { suggestionBanner.classList.add('hidden'); sendFeedback(-1.0); }); // ========================================== // 7. Policy Feedback Loop & Learning // ========================================== const acceptBtn = document.getElementById('acceptBtn'); const correctBtn = document.getElementById('correctBtn'); const correctionControlsEl = document.getElementById('correctionControls'); const previewCorrectionBtn = document.getElementById('previewCorrectionBtn'); const confirmCorrectionBtn = document.getElementById('confirmCorrectionBtn'); const cancelCorrectionBtn = document.getElementById('cancelCorrectionBtn'); const jsonValidationEl = document.getElementById('jsonValidation'); acceptBtn.addEventListener('click', () => { resetCorrectionState(); sendFeedback(1.0); }); correctBtn.addEventListener('click', enterCorrectionMode); if (previewCorrectionBtn) previewCorrectionBtn.addEventListener('click', previewCorrection); if (confirmCorrectionBtn) confirmCorrectionBtn.addEventListener('click', confirmCorrectionAndReinforce); if (cancelCorrectionBtn) cancelCorrectionBtn.addEventListener('click', cancelCorrection); // Live JSON validation for better correction UX if (jsonBox && jsonValidationEl) { jsonBox.addEventListener('input', () => { if (!jsonValidationEl) return; const val = jsonBox.value.trim(); if (!val) { jsonValidationEl.textContent = ''; jsonValidationEl.className = 'json-validation'; return; } try { const parsed = JSON.parse(val); if (Array.isArray(parsed)) { jsonValidationEl.textContent = '✓ Valid action array (' + parsed.length + ' step(s)) — safe to Preview'; jsonValidationEl.className = 'json-validation ok'; } else { jsonValidationEl.textContent = '⚠ Must be a plain JSON array of actions (edit only the [ ... ] part, no canonical wrapper)'; jsonValidationEl.className = 'json-validation bad'; } } catch (e) { jsonValidationEl.textContent = '✗ Invalid JSON: ' + e.message; jsonValidationEl.className = 'json-validation bad'; } }); } async function sendFeedback(rewardVal, isCorrection = false) { if (!lastTranscription) return; let reqData = { query: lastTranscription, action_json: JSON.stringify(lastInterpretedAction), reward: rewardVal > 0 ? 1.0 : 0.0 }; if (isCorrection) { try { const editedAction = JSON.parse(jsonBox.value); reqData.corrected_action_json = JSON.stringify(editedAction); // Pop the old unreinforced shape and draw the corrected action array const popCount = lastInterpretedAction.length; for (let i = 0; i < popCount; i++) { shapes.pop(); } editedAction.forEach(act => shapes.push(act)); renderAllShapes(); } catch (e) { alert("Invalid JSON array syntax. Ensure it is formatted as [ { ... } ]"); return; } } try { const response = await fetch('/api/reinforce', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(reqData) }); if (!response.ok) throw new Error("Reinforce API update failed"); const result = await response.json(); recordingStatus.textContent = result.message; loadMemories(); rewardCard.classList.add('disabled'); } catch (err) { console.error("Error committing feedback reward:", err); } } // ========================================== // 7b. Improved MemRL Correction Flow (preview sample before +1.0 DB update) // ========================================== function showCorrectionControls(show) { if (correctionControlsEl) { if (show) { correctionControlsEl.classList.remove('hidden'); } else { correctionControlsEl.classList.add('hidden'); if (confirmCorrectionBtn) confirmCorrectionBtn.disabled = true; if (jsonValidationEl) jsonValidationEl.textContent = ''; } } } function resetCorrectionState() { correctionSnapshot = null; showCorrectionControls(false); } function enterCorrectionMode() { if (!lastTranscription || !jsonBox) return; // Snapshot current canvas state so we can revert on cancel or re-preview correctionSnapshot = JSON.parse(JSON.stringify(shapes)); // Make sure the jsonBox has the current (possibly bad) interpretation for editing if (!jsonBox.value && lastInterpretedAction && lastInterpretedAction.length) { jsonBox.value = JSON.stringify(lastInterpretedAction, null, 2); } showCorrectionControls(true); recordingStatus.textContent = "Correction mode: edit the JSON above, click Preview to test drawing on the canvas. Only Accept when the sample looks right."; // Trigger a validation run jsonBox.dispatchEvent(new Event('input')); } function previewCorrection() { if (!jsonBox || !jsonBox.value || !correctionSnapshot) { recordingStatus.textContent = "No correction snapshot or JSON to preview."; return; } try { const editedAction = JSON.parse(jsonBox.value); if (!Array.isArray(editedAction)) throw new Error("JSON must be a plain array of actions — remove any 'canonical' wrapper if present"); // Always base the preview off the original snapshot (so repeated previews are clean) shapes = JSON.parse(JSON.stringify(correctionSnapshot)); // Apply the candidate corrected actions (same logic as before) editedAction.forEach(action => { if (action.shape === 'clear') { shapes = []; } else { shapes.push(action); } }); // Update runtime state so Accept knows what we approved lastInterpretedAction = editedAction; renderAllShapes(); if (confirmCorrectionBtn) confirmCorrectionBtn.disabled = false; recordingStatus.textContent = "Sample execution shown on canvas. Review the result. Click Accept to store this mapping in MemRL with +1.0 reinforcement."; } catch (e) { if (jsonValidationEl) { jsonValidationEl.textContent = '✗ ' + e.message; jsonValidationEl.className = 'json-validation bad'; } else { alert("Invalid JSON for preview: " + e.message); } } } async function confirmCorrectionAndReinforce() { if (!lastTranscription || !lastInterpretedAction || !correctionSnapshot) return; // At this point the canvas already shows the previewed/good result. // Send with +1.0 (positive reinforcement) and provide corrected_action_json // so backend can ensure the *good* action is the one associated with the (noisy) query. let reqData = { query: lastTranscription, action_json: JSON.stringify(lastInterpretedAction), reward: 1.0, corrected_action_json: JSON.stringify(lastInterpretedAction) }; try { const response = await fetch('/api/reinforce', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(reqData) }); if (!response.ok) throw new Error("Reinforce API update failed"); const result = await response.json(); recordingStatus.textContent = result.message || "Correction accepted with positive reinforcement."; loadMemories(); rewardCard.classList.add('disabled'); // Clean up correction mode (keep the good drawing on canvas) correctionSnapshot = null; showCorrectionControls(false); } catch (err) { console.error("Error committing corrected reinforcement:", err); recordingStatus.textContent = "Failed to save correction to MemRL. You can try again."; } } function cancelCorrection() { if (correctionSnapshot) { shapes = correctionSnapshot; renderAllShapes(); } // Restore the original interpreted action in the box if we have it if (lastInterpretedAction && jsonBox) { jsonBox.value = JSON.stringify(lastInterpretedAction, null, 2); } correctionSnapshot = null; showCorrectionControls(false); recordingStatus.textContent = "Correction cancelled. Canvas reverted to previous state."; } // ========================================== // 8. Q-Memory Database Loader // ========================================== const memoryTableBody = document.getElementById('memoryTableBody'); const resetMemoryBtn = document.getElementById('resetMemoryBtn'); resetMemoryBtn.addEventListener('click', wipeDatabase); async function loadMemories() { try { const response = await fetch('/api/memories'); if (!response.ok) throw new Error("Failed to load DB memories"); const data = await response.json(); populateMemoryTable(data.memories); } catch (err) { console.error("Error reading Q-database:", err); } } function populateMemoryTable(memories) { memoryTableBody.innerHTML = ''; if (!memories || memories.length === 0) { memoryTableBody.innerHTML = ` No learned memories yet. Use voice activation to begin. `; return; } memories.forEach(mem => { const row = document.createElement('tr'); let qClass = 'q-low'; if (mem.q_value >= 0.8) qClass = 'q-high'; else if (mem.q_value >= 0.4) qClass = 'q-mid'; const actionStr = JSON.stringify(mem.action); const lang = mem.language || 'multi'; const canon = mem.canonical_intent ? ` [${mem.canonical_intent}]` : ''; row.innerHTML = ` "${mem.query}" (${lang})${canon} ${actionStr} ${mem.q_value.toFixed(3)} `; memoryTableBody.appendChild(row); }); } async function wipeDatabase() { if (!confirm("Are you sure you want to reset all Q-values? This deletes all custom mapping history.")) return; try { const response = await fetch('/api/clear_memories', { method: 'POST' }); if (!response.ok) throw new Error("Wipe failed"); shapes = []; renderAllShapes(); lastTranscription = ""; lastInterpretedAction = []; transcriptionBox.textContent = "None"; jsonBox.value = ""; updateBadge("IDLE", "state-idle"); rewardCard.classList.add('disabled'); resetCorrectionState(); recordingStatus.textContent = "Database reset complete."; loadMemories(); } catch (err) { console.error("Error clearing SQLite DB:", err); } } // ========================================== // 9. Manual Palette & Settings Callbacks // ========================================== const toolBtns = document.querySelectorAll('.tool-btn'); const colorBtns = document.querySelectorAll('.color-btn'); const sizeSlider = document.getElementById('sizeSlider'); const sizeValText = document.getElementById('sizeVal'); const customColorPicker = document.getElementById('customColorPicker'); const manualClearBtn = document.getElementById('clearBtn'); const gridToggle = document.getElementById('gridToggle'); const animateToggle = document.getElementById('animateToggle'); // Dynamic listeners gridToggle.addEventListener('change', renderAllShapes); animateToggle.addEventListener('change', renderAllShapes); toolBtns.forEach(btn => { btn.addEventListener('click', () => { toolBtns.forEach(b => b.classList.remove('active')); btn.classList.add('active'); currentTool = btn.dataset.tool; }); }); colorBtns.forEach(btn => { btn.addEventListener('click', () => { colorBtns.forEach(b => b.classList.remove('active')); btn.classList.add('active'); currentColor = btn.dataset.color; customColorPicker.value = translateColorToHex(currentColor); }); }); customColorPicker.addEventListener('input', (e) => { colorBtns.forEach(b => b.classList.remove('active')); currentColor = e.target.value; }); sizeSlider.addEventListener('input', (e) => { strokeSize = parseInt(e.target.value); sizeValText.textContent = `${strokeSize}px`; }); manualClearBtn.addEventListener('click', clearCanvas); function translateColorToHex(cName) { const hexMap = { red: '#ff0000', blue: '#0000ff', green: '#008000', yellow: '#ffff00', purple: '#800080', orange: '#ffa500', black: '#000000' }; return hexMap[cName] || '#000000'; } // Manual drawing at click point canvas.addEventListener('click', (event) => { if (draggedShape) return; // avoid drawing when finishing a drag const rect = canvas.getBoundingClientRect(); const x = event.clientX - rect.left; const y = event.clientY - rect.top; // Renders shape centered at click point let clickAction = { shape: currentTool, color: currentColor, size: strokeSize, x: x, y: y }; if (currentTool === 'text') { const label = window.prompt('Text label?', 'label') || 'label'; clickAction.text = label; clickAction.size = Math.max(24, Math.min(strokeSize, 120)); } executeCanvasActions([clickAction]); }); // Setup click handlers for example pills const examplePills = document.querySelectorAll('.example-cmd-pill'); examplePills.forEach(pill => { pill.addEventListener('click', () => { const text = pill.textContent.replace(/"/g, '').trim(); // Emulate receiving this text from speech runMockTextCommand(text); }); }); async function runMockTextCommand(text) { recordingStatus.textContent = "Processing mock command..."; transcriptionBox.textContent = `"${text}"`; updateBadge("PROCESSING", "state-suggest"); // Demo path for example pills: client-side memory lookup (levenshtein) + local rule fallback. // This is fast, fully offline, and lets users test + immediately reinforce via the feedback UI. // (We fire a dummy to /api/transcribe only for side-effect consistency; result is ignored.) try { // Fire-and-forget dummy to keep API path warm (optional) fetch(`/api/transcribe`, { method: 'POST', body: createFormWithDummyAudio() }).catch(() => {}); lastTranscription = text; // Query memory locally by sending query? No, let's fetch memories list and look up: const memResponse = await fetch('/api/memories'); const memData = await memResponse.json(); let match = null; let max_sim = 0.0; // Custom js levenshtein logic const threshold = 0.75; memData.memories.forEach(mem => { const sim = levenshtein_similarity(text, mem.query); if (sim > max_sim) { max_sim = sim; match = mem; } }); if (max_sim >= threshold && match) { resetCorrectionState(); lastInterpretedAction = match.action; jsonBox.value = JSON.stringify(lastInterpretedAction, null, 2); rewardCard.classList.remove('disabled'); if (match.q_value >= 0.8) { updateBadge("MEMRL RECALL (1.0)", "state-matched"); executeCanvasActions(lastInterpretedAction); recordingStatus.textContent = "Auto-executed from memory!"; } else { updateBadge("SUGGESTION MAP", "state-suggest"); showSuggestionBanner(text, lastInterpretedAction); recordingStatus.textContent = "Suggestion found in memory."; } } else { // Fallback rule based parse locally to render instantly! // This ensures instant demo feedback for pills. resetCorrectionState(); const localFallback = localRuleBasedFallback(text); lastInterpretedAction = localFallback; jsonBox.value = JSON.stringify(lastInterpretedAction, null, 2); rewardCard.classList.remove('disabled'); updateBadge("LOCAL INTERPRETER", "state-miss"); executeCanvasActions(lastInterpretedAction); recordingStatus.textContent = "Resolved via local parser."; } } catch (e) { console.error("Mock pill run failed:", e); } } function createFormWithDummyAudio() { const dummyBlob = new Blob([new Uint8Array(100)], { type: 'audio/wav' }); const fd = new FormData(); fd.append('audio', dummyBlob, 'dummy.wav'); return fd; } function levenshtein_similarity(s1, s2) { // Match Python implementation in app.py exactly for symmetry (per Agents.md): // lower + strip whitespace + strip punctuation chars from ends. s1 = (s1 || "").toLowerCase().trim().replace(/^[.?,!]+|[.?,!]+$/g, ''); s2 = (s2 || "").toLowerCase().trim().replace(/^[.?,!]+|[.?,!]+$/g, ''); if (s1 === s2) return 1.0; const m = s1.length, n = s2.length; if (Math.max(m, n) === 0) return 1.0; const dp = Array(m + 1).fill().map(() => Array(n + 1).fill(0)); for (let i = 0; i <= m; i++) dp[i][0] = i; for (let j = 0; j <= n; j++) dp[0][j] = j; for (let i = 1; i <= m; i++) { for (let j = 1; j <= n; j++) { if (s1[i-1] === s2[j-1]) dp[i][j] = dp[i-1][j-1]; else dp[i][j] = Math.min(dp[i-1][j], dp[i][j-1], dp[i-1][j-1]) + 1; } } return 1.0 - (dp[m][n] / Math.max(m, n)); } // Client-side rule based fallback matching localRuleBasedFallback function localRuleBasedFallback(user_text) { const lower_text = user_text.toLowerCase(); const parts = lower_text.split(/\band\b|\bthen\b|,/); const actions = []; const shapes_list = ["circle", "square", "rectangle", "triangle", "star", "pentagon", "hexagon", "oval", "heart", "line", "arrow", "text"]; // Multilingual color map for demo (Spanish/French etc.) const color_map = { "red": ["red", "rojo", "rouge"], "blue": ["blue", "azul", "bleu"], "green": ["green", "verde", "vert"], "yellow": ["yellow", "amarillo", "jaune"], "orange": ["orange", "naranja"], "purple": ["purple", "morado", "violet", "pourpre"], "pink": ["pink", "rosa", "rose"], "black": ["black", "negro", "noir"], "white": ["white", "blanco", "blanc"], "cyan": ["cyan"], "magenta": ["magenta"], }; parts.forEach(part => { part = part.trim(); if (!part) return; if (part.includes("clear") || part.includes("wipe") || part.includes("reset")) { actions.push({shape: 'clear', color: 'white', size: 0}); return; } let color = "black"; for (let eng in color_map) { if (color_map[eng].some(v => part.includes(v))) { color = eng; break; } } let size = 100; const size_match = part.match(/\b(\d+)\b/); if (size_match) { const val = parseInt(size_match[1]); if (val >= 30 && val <= 300) size = val; } let px = "center"; if (part.includes("left")) px = "left"; else if (part.includes("right")) px = "right"; let py = "center"; if (part.includes("top")) py = "top"; else if (part.includes("bottom")) py = "bottom"; // Text label (write, say, text, label) - per Agents.md roadmap const textMatch = part.match(/["']([^"']+)["']/); const isTextCmd = /text|write|say|label|title/.test(part); if (isTextCmd) { const label = textMatch ? textMatch[1].trim() : (part.split(/\s+/).pop() || "label"); const fontSize = Math.max(24, Math.min(size, 160)); actions.push({ shape: 'text', text: label, color: color, size: fontSize, x: px, y: py }); return; } // Check for relative paths if (part.includes("path") || part.includes("freehand") || part.includes("go") || part.includes("move")) { const operations = []; operations.push({type: 'start', x: px, y: py}); const words = part.split(/\s+/); words.forEach((word, idx) => { if (["right", "east"].includes(word) && idx + 1 < words.length) { const val = parseInt(words[idx+1].replace(/[^0-9]/g, '')); if (!isNaN(val)) operations.push({type: 'line', dx: val, dy: 0}); } else if (["left", "west"].includes(word) && idx + 1 < words.length) { const val = parseInt(words[idx+1].replace(/[^0-9]/g, '')); if (!isNaN(val)) operations.push({type: 'line', dx: -val, dy: 0}); } else if (["down", "south"].includes(word) && idx + 1 < words.length) { const val = parseInt(words[idx+1].replace(/[^0-9]/g, '')); if (!isNaN(val)) operations.push({type: 'line', dx: 0, dy: val}); } else if (["up", "north"].includes(word) && idx + 1 < words.length) { const val = parseInt(words[idx+1].replace(/[^0-9]/g, '')); if (!isNaN(val)) operations.push({type: 'line', dx: 0, dy: -val}); } else if (word === "dot" || word === "point") { operations.push({type: 'dot', radius: 8}); } }); if (operations.length > 1) { actions.push({ shape: 'path', color: color, thickness: 5, operations: operations }); } return; } let shape_found = null; shapes_list.forEach(s => { if (part.includes(s)) shape_found = s; }); if (shape_found === "line" || shape_found === "arrow") { const start_x = part.includes("left") ? "left" : "center"; const start_y = part.includes("top") ? "top" : "center"; const end_x = part.includes("right") ? "right" : 350; const end_y = part.includes("bottom") ? "bottom" : 350; actions.push({ shape: shape_found, color: color, start_x: start_x, start_y: start_y, end_x: end_x, end_y: end_y }); } else if (shape_found) { actions.push({ shape: shape_found, color: color, size: size, x: px, y: py }); } else { actions.push({ shape: 'circle', color: color, size: size, x: px, y: py }); } }); return actions; } // Initialize on load window.addEventListener('DOMContentLoaded', () => { initCanvas(); loadMemories(); });