memrl-canvas / static /script.js
indroniel
fix(multilingual): Improve color extraction for Spanish/French in both server rule-based and client-side fallback
709a035
Raw
History Blame Contribute Delete
50.5 kB
// ==========================================================================
// MEMRL CANVAS FRONTEND LOGIC (ADVANCED V2)
// ==========================================================================
// Drawing State
let currentTool = 'circle';
let currentColor = 'red';
let strokeSize = 100;
let shapes = [];
let undoStack = [];
// Drag and drop state
let draggedShape = null;
let draggedIndex = -1;
let dragOffsetX = 0;
let dragOffsetY = 0;
// Audio recording state
let mediaRecorder = null;
let audioChunks = [];
let audioContext = null;
let analyser = null;
let animationFrameId = null;
let isRecording = false;
// MemRL state variables
let lastTranscription = "";
let lastInterpretedAction = [];
// Correction mode state (better preview-before-reinforce flow for MemRL)
let correctionSnapshot = null;
let correctionControls = null;
// Tracing animation state
let animationProgress = 1.0;
let animationActive = false;
// ==========================================
// 1. Coordinate Mapping & Shape Drawing Math
// ==========================================
const canvas = document.getElementById('drawingCanvas');
const ctx = canvas.getContext('2d');
function resolveX(val) {
if (val === 'left') return 120;
if (val === 'right') return 380;
if (val === 'center') return 250;
if (typeof val === 'number') return val;
const num = parseFloat(val);
return isNaN(num) ? 250 : num;
}
function resolveY(val) {
if (val === 'top') return 120;
if (val === 'bottom') return 380;
if (val === 'center') return 250;
if (typeof val === 'number') return val;
const num = parseFloat(val);
return isNaN(num) ? 250 : num;
}
// Draw a blueprint engineering grid background
function drawGrid() {
if (!document.getElementById('gridToggle').checked) return;
ctx.strokeStyle = '#e2e8f0';
ctx.lineWidth = 1;
// Vertical grid lines
for (let x = 25; x < canvas.width; x += 25) {
ctx.beginPath();
ctx.moveTo(x, 0);
ctx.lineTo(x, canvas.height);
ctx.strokeStyle = (x % 100 === 0) ? '#cbd5e1' : '#f1f5f9';
ctx.stroke();
}
// Horizontal grid lines
for (let y = 25; y < canvas.height; y += 25) {
ctx.beginPath();
ctx.moveTo(0, y);
ctx.lineTo(canvas.width, y);
ctx.strokeStyle = (y % 100 === 0) ? '#cbd5e1' : '#f1f5f9';
ctx.stroke();
}
}
// Math equations for drawing academic shapes
function drawCircle(ctx, cx, cy, size, drawStroke) {
ctx.beginPath();
ctx.arc(cx, cy, size / 2, 0, Math.PI * 2);
if (drawStroke) ctx.stroke();
else ctx.fill();
}
function drawSquare(ctx, cx, cy, size, drawStroke) {
ctx.beginPath();
ctx.rect(cx - size / 2, cy - size / 2, size, size);
if (drawStroke) ctx.stroke();
else ctx.fill();
}
function drawRectangle(ctx, cx, cy, size, drawStroke) {
ctx.beginPath();
ctx.rect(cx - size, cy - size / 2, size * 2, size);
if (drawStroke) ctx.stroke();
else ctx.fill();
}
function drawTriangle(ctx, cx, cy, size, drawStroke) {
ctx.beginPath();
ctx.moveTo(cx, cy - size / 2);
ctx.lineTo(cx - size / 2, cy + size / 2);
ctx.lineTo(cx + size / 2, cy + size / 2);
ctx.closePath();
if (drawStroke) ctx.stroke();
else ctx.fill();
}
function drawPentagon(ctx, cx, cy, size, drawStroke) {
ctx.beginPath();
const r = size / 2;
for (let i = 0; i < 5; i++) {
const angle = (i * 2 * Math.PI / 5) - Math.PI / 2;
const x = cx + r * Math.cos(angle);
const y = cy + r * Math.sin(angle);
if (i === 0) ctx.moveTo(x, y);
else ctx.lineTo(x, y);
}
ctx.closePath();
if (drawStroke) ctx.stroke();
else ctx.fill();
}
function drawHexagon(ctx, cx, cy, size, drawStroke) {
ctx.beginPath();
const r = size / 2;
for (let i = 0; i < 6; i++) {
const angle = (i * 2 * Math.PI / 6) - Math.PI / 2;
const x = cx + r * Math.cos(angle);
const y = cy + r * Math.sin(angle);
if (i === 0) ctx.moveTo(x, y);
else ctx.lineTo(x, y);
}
ctx.closePath();
if (drawStroke) ctx.stroke();
else ctx.fill();
}
function drawStar(ctx, cx, cy, size, drawStroke) {
ctx.beginPath();
const rOuter = size / 2;
const rInner = size / 4.5;
const points = 5;
for (let i = 0; i < 2 * points; i++) {
const angle = (i * Math.PI / points) - Math.PI / 2;
const r = (i % 2 === 0) ? rOuter : rInner;
const x = cx + r * Math.cos(angle);
const y = cy + r * Math.sin(angle);
if (i === 0) ctx.moveTo(x, y);
else ctx.lineTo(x, y);
}
ctx.closePath();
if (drawStroke) ctx.stroke();
else ctx.fill();
}
function drawOval(ctx, cx, cy, size, drawStroke) {
ctx.beginPath();
ctx.ellipse(cx, cy, size / 2, size / 3, 0, 0, Math.PI * 2);
if (drawStroke) ctx.stroke();
else ctx.fill();
}
function drawHeart(ctx, cx, cy, size, drawStroke) {
ctx.beginPath();
const d = size * 0.72;
const x = cx;
const y = cy - d / 4;
ctx.moveTo(x, y);
ctx.bezierCurveTo(x - d / 2, y - d / 2, x - d, y + d / 6, x, y + d * 0.7);
ctx.bezierCurveTo(x + d, y + d / 6, x + d / 2, y - d / 2, x, y);
ctx.closePath();
if (drawStroke) ctx.stroke();
else ctx.fill();
}
function drawTextShape(ctx, shape, progress = 1.0) {
const tx = resolveX(shape.x !== undefined ? shape.x : 'center');
const ty = resolveY(shape.y !== undefined ? shape.y : 'center');
const fsize = shape.size || 48;
const label = shape.text || 'text';
ctx.save();
ctx.fillStyle = shape.color || 'black';
ctx.font = `${fsize}px sans-serif`;
ctx.textAlign = 'center';
ctx.textBaseline = 'middle';
// Simple fade-in for animation consistency
ctx.globalAlpha = Math.min(1.0, Math.max(0.2, progress));
ctx.fillText(label, tx, ty);
ctx.restore();
}
function drawLineShape(ctx, startX, startY, endX, endY, progress = 1.0) {
ctx.beginPath();
ctx.moveTo(startX, startY);
// Draw progressive line length based on animation progress
const curX = startX + (endX - startX) * progress;
const curY = startY + (endY - startY) * progress;
ctx.lineTo(curX, curY);
ctx.stroke();
}
function drawArrowShape(ctx, startX, startY, endX, endY, progress = 1.0) {
ctx.beginPath();
ctx.moveTo(startX, startY);
const curX = startX + (endX - startX) * progress;
const curY = startY + (endY - startY) * progress;
ctx.lineTo(curX, curY);
ctx.stroke();
// Draw arrowhead only at the end
if (progress >= 0.9) {
const angle = Math.atan2(endY - startY, endX - startX);
const headLength = 14;
ctx.beginPath();
ctx.moveTo(endX, endY);
ctx.lineTo(endX - headLength * Math.cos(angle - Math.PI / 6), endY - headLength * Math.sin(angle - Math.PI / 6));
ctx.lineTo(endX - headLength * Math.cos(angle + Math.PI / 6), endY - headLength * Math.sin(angle + Math.PI / 6));
ctx.closePath();
ctx.fill();
}
}
// Path / Freehand Operations Runner
function drawPathShape(ctx, shape, drawStroke, progress = 1.0) {
const ops = shape.operations || [];
if (ops.length === 0) return;
const opsToDraw = Math.floor(ops.length * progress);
let px = resolveX('center');
let py = resolveY('center');
ctx.beginPath();
ops.forEach((op, index) => {
if (index > opsToDraw) return;
if (op.type === 'start') {
px = resolveX(op.x);
py = resolveY(op.y);
ctx.moveTo(px, py);
} else if (op.type === 'line') {
px += op.dx || 0;
py += op.dy || 0;
ctx.lineTo(px, py);
} else if (op.type === 'move') {
px += op.dx || 0;
py += op.dy || 0;
ctx.moveTo(px, py);
} else if (op.type === 'dot') {
const r = op.radius || 6;
ctx.arc(px, py, r, 0, Math.PI * 2);
ctx.fill();
ctx.beginPath();
ctx.moveTo(px, py);
}
});
if (opsToDraw > 0) {
ctx.stroke();
}
}
// ==========================================
// 2. Render Pipeline
// ==========================================
function renderAllShapes() {
// Fill canvas background
ctx.fillStyle = '#ffffff';
ctx.fillRect(0, 0, canvas.width, canvas.height);
// Draw blueprint lines
drawGrid();
// Draw shapes in order
shapes.forEach((shape, index) => {
const isLast = (index === shapes.length - 1);
const progress = (isLast && animationActive) ? animationProgress : 1.0;
ctx.strokeStyle = shape.color || 'black';
ctx.fillStyle = shape.color || 'black';
ctx.lineWidth = shape.thickness || 4;
ctx.lineCap = 'round';
ctx.lineJoin = 'round';
const cx = resolveX(shape.x !== undefined ? shape.x : 'center');
const cy = resolveY(shape.y !== undefined ? shape.y : 'center');
const size = shape.size || 100;
// 1. Path-based drawings
if (shape.shape === 'path') {
drawPathShape(ctx, shape, true, progress);
return;
}
// 2. Lines & Arrows
if (shape.shape === 'line') {
const sx = resolveX(shape.start_x !== undefined ? shape.start_x : 'center');
const sy = resolveY(shape.start_y !== undefined ? shape.start_y : 'center');
const ex = resolveX(shape.end_x !== undefined ? shape.end_x : 350);
const ey = resolveY(shape.end_y !== undefined ? shape.end_y : 350);
drawLineShape(ctx, sx, sy, ex, ey, progress);
return;
}
if (shape.shape === 'arrow') {
const sx = resolveX(shape.start_x !== undefined ? shape.start_x : 'center');
const sy = resolveY(shape.start_y !== undefined ? shape.start_y : 'center');
const ex = resolveX(shape.end_x !== undefined ? shape.end_x : 350);
const ey = resolveY(shape.end_y !== undefined ? shape.end_y : 350);
drawArrowShape(ctx, sx, sy, ex, ey, progress);
return;
}
// 3. Clear canvas command
if (shape.shape === 'clear') {
ctx.fillStyle = '#ffffff';
ctx.fillRect(0, 0, canvas.width, canvas.height);
drawGrid();
return;
}
// 3.5 Text labels (Agents.md roadmap item)
if (shape.shape === 'text') {
const tprog = (isLast && animationActive) ? animationProgress : 1.0;
drawTextShape(ctx, shape, tprog);
return;
}
// 4. Standard Geometry Tracing Animation logic
if (progress < 0.6) {
// Step 1: Trace outlines only (line dash animated)
ctx.save();
const strokeProgress = progress / 0.6;
ctx.setLineDash([size * 3 * strokeProgress, size * 3]);
drawGeometry(ctx, cx, cy, size, shape.shape, true);
ctx.restore();
} else {
// Step 2: Flood fill opacity
ctx.save();
ctx.globalAlpha = (progress - 0.6) / 0.4;
drawGeometry(ctx, cx, cy, size, shape.shape, false);
ctx.restore();
// Draw stroke outer bounds
ctx.lineWidth = 2;
drawGeometry(ctx, cx, cy, size, shape.shape, true);
}
});
}
function drawGeometry(ctx, cx, cy, size, shapeType, drawStroke) {
if (shapeType === 'circle') drawCircle(ctx, cx, cy, size, drawStroke);
else if (shapeType === 'square') drawSquare(ctx, cx, cy, size, drawStroke);
else if (shapeType === 'rectangle') drawRectangle(ctx, cx, cy, size, drawStroke);
else if (shapeType === 'triangle') drawTriangle(ctx, cx, cy, size, drawStroke);
else if (shapeType === 'pentagon') drawPentagon(ctx, cx, cy, size, drawStroke);
else if (shapeType === 'hexagon') drawHexagon(ctx, cx, cy, size, drawStroke);
else if (shapeType === 'star') drawStar(ctx, cx, cy, size, drawStroke);
else if (shapeType === 'oval') drawOval(ctx, cx, cy, size, drawStroke);
else if (shapeType === 'heart') drawHeart(ctx, cx, cy, size, drawStroke);
}
function triggerTraceAnimation() {
if (!document.getElementById('animateToggle').checked) {
animationProgress = 1.0;
animationActive = false;
renderAllShapes();
return;
}
animationProgress = 0.0;
animationActive = true;
animateLoop();
}
function animateLoop() {
if (!animationActive) return;
animationProgress += 0.04;
if (animationProgress >= 1.0) {
animationProgress = 1.0;
animationActive = false;
}
renderAllShapes();
if (animationActive) {
requestAnimationFrame(animateLoop);
}
}
function executeCanvasActions(actionList) {
if (!actionList || !Array.isArray(actionList)) return;
undoStack.push(JSON.parse(JSON.stringify(shapes)));
actionList.forEach(action => {
if (action.shape === 'clear') {
shapes = [];
} else {
shapes.push(action);
}
});
triggerTraceAnimation();
}
// ==========================================
// 3. Dynamic Shape Drag & Drop Hit Detection
// ==========================================
function isPointInShape(px, py, shape) {
const cx = resolveX(shape.x !== undefined ? shape.x : 'center');
const cy = resolveY(shape.y !== undefined ? shape.y : 'center');
const size = shape.size || 100;
if (shape.shape === 'circle') {
const dist = Math.sqrt((px - cx) ** 2 + (py - cy) ** 2);
return dist <= size / 2;
} else if (shape.shape === 'square') {
return px >= cx - size / 2 && px <= cx + size / 2 && py >= cy - size / 2 && py <= cy + size / 2;
} else if (shape.shape === 'rectangle') {
return px >= cx - size && px <= cx + size && py >= cy - size / 2 && py <= cy + size / 2;
} else if (shape.shape === 'path') {
// Simple bounding box for paths based on center coordinate
return px >= cx - 50 && px <= cx + 50 && py >= cy - 50 && py <= cy + 50;
} else if (['triangle', 'star', 'pentagon', 'hexagon', 'oval', 'heart'].includes(shape.shape)) {
// Bounding box approximation
return px >= cx - size / 2 && px <= cx + size / 2 && py >= cy - size / 2 && py <= cy + size / 2;
} else if (shape.shape === 'text') {
// Approximate text hit box (centered). Width ~ 0.5 * size per char, height ~ 0.8 * size.
const label = shape.text || 'text';
const approxW = Math.max(30, label.length * (size * 0.45));
const approxH = size * 0.8;
return px >= cx - approxW / 2 && px <= cx + approxW / 2 && py >= cy - approxH / 2 && py <= cy + approxH / 2;
} else if (shape.shape === 'line' || shape.shape === 'arrow') {
// Distance from point to line segment formula
const sx = resolveX(shape.start_x !== undefined ? shape.start_x : 'center');
const sy = resolveY(shape.start_y !== undefined ? shape.start_y : 'center');
const ex = resolveX(shape.end_x !== undefined ? shape.end_x : 350);
const ey = resolveY(shape.end_y !== undefined ? shape.end_y : 350);
const A = px - sx;
const B = py - sy;
const C = ex - sx;
const D = ey - sy;
const dot = A * C + B * D;
const lenSq = C * C + D * D;
let param = -1;
if (lenSq !== 0) param = dot / lenSq;
let xx, yy;
if (param < 0) {
xx = sx;
yy = sy;
} else if (param > 1) {
xx = ex;
yy = ey;
} else {
xx = sx + param * C;
yy = sy + param * D;
}
const dist = Math.sqrt((px - xx) ** 2 + (py - yy) ** 2);
return dist <= 12; // hit tolerance of 12 pixels
}
return false;
}
canvas.addEventListener('mousedown', (e) => {
const rect = canvas.getBoundingClientRect();
const mx = e.clientX - rect.left;
const my = e.clientY - rect.top;
for (let i = shapes.length - 1; i >= 0; i--) {
const shape = shapes[i];
if (isPointInShape(mx, my, shape)) {
draggedShape = shape;
draggedIndex = i;
// Set drag offsets
const cx = resolveX(shape.x !== undefined ? shape.x : 'center');
const cy = resolveY(shape.y !== undefined ? shape.y : 'center');
dragOffsetX = mx - cx;
dragOffsetY = my - cy;
canvas.style.cursor = 'grabbing';
break;
}
}
});
canvas.addEventListener('mousemove', (e) => {
const rect = canvas.getBoundingClientRect();
const mx = e.clientX - rect.left;
const my = e.clientY - rect.top;
if (draggedShape) {
// Dragging line requires moving start and endpoints relatively
if (draggedShape.shape === 'line' || draggedShape.shape === 'arrow') {
const sx = resolveX(draggedShape.start_x !== undefined ? draggedShape.start_x : 'center');
const sy = resolveY(draggedShape.start_y !== undefined ? draggedShape.start_y : 'center');
const ex = resolveX(draggedShape.end_x !== undefined ? draggedShape.end_x : 350);
const ey = resolveY(draggedShape.end_y !== undefined ? draggedShape.end_y : 350);
const dx = mx - (sx + ex) / 2;
const dy = my - (sy + ey) / 2;
draggedShape.start_x = sx + dx;
draggedShape.start_y = sy + dy;
draggedShape.end_x = ex + dx;
draggedShape.end_y = ey + dy;
} else {
// Standard shape movement coordinates mapping
draggedShape.x = mx - dragOffsetX;
draggedShape.y = my - dragOffsetY;
}
renderAllShapes();
} else {
// Hover cursor styling
let hover = false;
for (let i = shapes.length - 1; i >= 0; i--) {
if (isPointInShape(mx, my, shapes[i])) {
hover = true;
break;
}
}
canvas.style.cursor = hover ? 'grab' : 'crosshair';
}
});
canvas.addEventListener('mouseup', () => {
if (draggedShape) {
draggedShape = null;
canvas.style.cursor = 'grab';
// Sync manual parameter box with newly dragged coordinates
jsonBox.value = JSON.stringify(shapes, null, 2);
}
});
// ==========================================
// 4. Audio Controls & Recording
// ==========================================
const micBtn = document.getElementById('micBtn');
const waveContainer = document.getElementById('recordingWave');
const recordingStatus = document.getElementById('recordingStatus');
micBtn.addEventListener('click', toggleRecording);
async function toggleRecording() {
if (isRecording) {
stopRecording();
} else {
await startRecording();
}
}
async function startRecording() {
audioChunks = [];
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
let options = { mimeType: 'audio/webm' };
if (!MediaRecorder.isTypeSupported('audio/webm')) {
options = { mimeType: 'audio/ogg' };
if (!MediaRecorder.isTypeSupported('audio/ogg')) {
options = { mimeType: 'audio/mp4' };
}
}
mediaRecorder = new MediaRecorder(stream, options);
mediaRecorder.ondataavailable = (e) => {
if (e.data.size > 0) audioChunks.push(e.data);
};
mediaRecorder.onstop = sendAudioToBackend;
setupVisualizer(stream);
mediaRecorder.start();
isRecording = true;
micBtn.classList.add('recording');
waveContainer.classList.add('active');
recordingStatus.textContent = "Recording... Click again to process.";
} catch (err) {
console.error("Audio device access error:", err);
recordingStatus.textContent = "Error: Microphone access denied.";
}
}
function stopRecording() {
if (mediaRecorder && isRecording) {
mediaRecorder.stop();
mediaRecorder.stream.getTracks().forEach(track => track.stop());
isRecording = false;
micBtn.classList.remove('recording');
waveContainer.classList.remove('active');
recordingStatus.textContent = "Processing speech...";
if (animationFrameId) cancelAnimationFrame(animationFrameId);
if (audioContext) audioContext.close();
}
}
function setupVisualizer(stream) {
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
const source = audioContext.createMediaStreamSource(stream);
source.connect(analyser);
analyser.fftSize = 32;
const bufferLength = analyser.frequencyBinCount;
const dataArray = new Uint8Array(bufferLength);
const bars = document.querySelectorAll('.wave-bar');
function draw() {
if (!isRecording) return;
animationFrameId = requestAnimationFrame(draw);
analyser.getByteFrequencyData(dataArray);
bars.forEach((bar, index) => {
const value = dataArray[index % bufferLength];
const height = Math.max(4, Math.min(40, (value / 255) * 45));
bar.style.height = `${height}px`;
});
}
draw();
}
// ==========================================
// 5. API Loop Operations
// ==========================================
const transcriptionBox = document.getElementById('transcriptionBox');
const statusBadge = document.getElementById('statusBadge');
const jsonBox = document.getElementById('jsonBox');
const rewardCard = document.getElementById('rewardCard');
async function sendAudioToBackend() {
const audioBlob = new Blob(audioChunks, { type: mediaRecorder.mimeType });
const formData = new FormData();
formData.append('audio', audioBlob, 'voice_command.wav');
try {
const response = await fetch('/api/transcribe', {
method: 'POST',
body: formData
});
if (!response.ok) throw new Error("Transcription server error");
const result = await response.json();
processASRResult(result);
} catch (err) {
console.error("Transcribe API communication failed:", err);
recordingStatus.textContent = "Error transcribing audio command.";
}
}
function processASRResult(result) {
recordingStatus.textContent = "Command resolved.";
// Exit any pending correction session when a new voice command arrives
resetCorrectionState();
lastTranscription = result.transcription;
lastInterpretedAction = result.action;
transcriptionBox.textContent = lastTranscription ? `"${lastTranscription}"` : "None";
if (!lastTranscription || !lastInterpretedAction || lastInterpretedAction.length === 0) {
updateBadge("IDLE", "state-idle");
jsonBox.value = "";
rewardCard.classList.add('disabled');
return;
}
// Always keep jsonBox as pure actions array so correction/preview editing works
jsonBox.value = JSON.stringify(lastInterpretedAction, null, 2);
rewardCard.classList.remove('disabled');
// Extra safety for demo: if actions empty but we have canonical, synthesize on client too
if ((!lastInterpretedAction || lastInterpretedAction.length === 0) && result.canonical) {
const c = result.canonical;
const slots = c.slots || {};
lastInterpretedAction = [{
shape: slots.shape || (c.intent === 'draw_shape' ? 'circle' : c.intent) || 'circle',
color: slots.color || 'black',
size: slots.size || 100,
x: slots.x || 'center',
y: slots.y || 'center'
}];
if (c.intent === 'clear') {
lastInterpretedAction = [{ shape: 'clear', color: 'white', size: 0 }];
}
jsonBox.value = JSON.stringify(lastInterpretedAction, null, 2);
}
// Show canonical info in status for demo visibility (without polluting the editable JSON box)
if (result.canonical) {
const canonStr = JSON.stringify(result.canonical);
// Append a small note to transcription area for now (demo only)
const current = transcriptionBox.textContent || '';
if (!current.includes('Canonical:')) {
transcriptionBox.textContent = current + ` | Canonical: ${canonStr}`;
}
}
// Rich status for Hackathon demo (advertises multilingual + canonical)
const status = result.status || "";
if (status.includes("MemRL Template") || status.includes("Auto-Executed")) {
updateBadge("MEMRL TEMPLATE (lang-agnostic)", "state-matched");
executeCanvasActions(lastInterpretedAction);
} else if (status.includes("Low Confidence") || status.includes("Suggestion")) {
updateBadge("MEMRL SUGGESTION (multilingual)", "state-suggest");
showSuggestionBanner(lastTranscription, lastInterpretedAction);
} else if (status.includes("Gemma + RAG") || status.includes("canonical")) {
const langNote = result.language ? ` (${result.language})` : "";
updateBadge(`GEMMA+RAG (canonical${langNote})`, "state-miss");
executeCanvasActions(lastInterpretedAction);
} else {
updateBadge("INTERPRETED", "state-miss");
executeCanvasActions(lastInterpretedAction);
}
}
function updateBadge(text, className) {
statusBadge.textContent = text;
statusBadge.className = `status-badge ${className}`;
}
// ==========================================
// 6. Suggestion Overlay Banner
// ==========================================
const suggestionBanner = document.getElementById('suggestionBanner');
const bannerYes = document.getElementById('bannerYes');
const bannerNo = document.getElementById('bannerNo');
function showSuggestionBanner(query, actionList) {
const desc = `${actionList.length} action shape(s) resolved.`;
suggestionBanner.querySelector('.banner-text').textContent = `MemRL Suggestion: Did you mean to draw: "${query}"? (${desc})`;
suggestionBanner.classList.remove('hidden');
}
bannerYes.addEventListener('click', () => {
suggestionBanner.classList.add('hidden');
executeCanvasActions(lastInterpretedAction);
sendFeedback(1.0);
});
bannerNo.addEventListener('click', () => {
suggestionBanner.classList.add('hidden');
sendFeedback(-1.0);
});
// ==========================================
// 7. Policy Feedback Loop & Learning
// ==========================================
const acceptBtn = document.getElementById('acceptBtn');
const correctBtn = document.getElementById('correctBtn');
const correctionControlsEl = document.getElementById('correctionControls');
const previewCorrectionBtn = document.getElementById('previewCorrectionBtn');
const confirmCorrectionBtn = document.getElementById('confirmCorrectionBtn');
const cancelCorrectionBtn = document.getElementById('cancelCorrectionBtn');
const jsonValidationEl = document.getElementById('jsonValidation');
acceptBtn.addEventListener('click', () => {
resetCorrectionState();
sendFeedback(1.0);
});
correctBtn.addEventListener('click', enterCorrectionMode);
if (previewCorrectionBtn) previewCorrectionBtn.addEventListener('click', previewCorrection);
if (confirmCorrectionBtn) confirmCorrectionBtn.addEventListener('click', confirmCorrectionAndReinforce);
if (cancelCorrectionBtn) cancelCorrectionBtn.addEventListener('click', cancelCorrection);
// Live JSON validation for better correction UX
if (jsonBox && jsonValidationEl) {
jsonBox.addEventListener('input', () => {
if (!jsonValidationEl) return;
const val = jsonBox.value.trim();
if (!val) {
jsonValidationEl.textContent = '';
jsonValidationEl.className = 'json-validation';
return;
}
try {
const parsed = JSON.parse(val);
if (Array.isArray(parsed)) {
jsonValidationEl.textContent = '✓ Valid action array (' + parsed.length + ' step(s)) — safe to Preview';
jsonValidationEl.className = 'json-validation ok';
} else {
jsonValidationEl.textContent = '⚠ Must be a plain JSON array of actions (edit only the [ ... ] part, no canonical wrapper)';
jsonValidationEl.className = 'json-validation bad';
}
} catch (e) {
jsonValidationEl.textContent = '✗ Invalid JSON: ' + e.message;
jsonValidationEl.className = 'json-validation bad';
}
});
}
async function sendFeedback(rewardVal, isCorrection = false) {
if (!lastTranscription) return;
let reqData = {
query: lastTranscription,
action_json: JSON.stringify(lastInterpretedAction),
reward: rewardVal > 0 ? 1.0 : 0.0
};
if (isCorrection) {
try {
const editedAction = JSON.parse(jsonBox.value);
reqData.corrected_action_json = JSON.stringify(editedAction);
// Pop the old unreinforced shape and draw the corrected action array
const popCount = lastInterpretedAction.length;
for (let i = 0; i < popCount; i++) {
shapes.pop();
}
editedAction.forEach(act => shapes.push(act));
renderAllShapes();
} catch (e) {
alert("Invalid JSON array syntax. Ensure it is formatted as [ { ... } ]");
return;
}
}
try {
const response = await fetch('/api/reinforce', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(reqData)
});
if (!response.ok) throw new Error("Reinforce API update failed");
const result = await response.json();
recordingStatus.textContent = result.message;
loadMemories();
rewardCard.classList.add('disabled');
} catch (err) {
console.error("Error committing feedback reward:", err);
}
}
// ==========================================
// 7b. Improved MemRL Correction Flow (preview sample before +1.0 DB update)
// ==========================================
function showCorrectionControls(show) {
if (correctionControlsEl) {
if (show) {
correctionControlsEl.classList.remove('hidden');
} else {
correctionControlsEl.classList.add('hidden');
if (confirmCorrectionBtn) confirmCorrectionBtn.disabled = true;
if (jsonValidationEl) jsonValidationEl.textContent = '';
}
}
}
function resetCorrectionState() {
correctionSnapshot = null;
showCorrectionControls(false);
}
function enterCorrectionMode() {
if (!lastTranscription || !jsonBox) return;
// Snapshot current canvas state so we can revert on cancel or re-preview
correctionSnapshot = JSON.parse(JSON.stringify(shapes));
// Make sure the jsonBox has the current (possibly bad) interpretation for editing
if (!jsonBox.value && lastInterpretedAction && lastInterpretedAction.length) {
jsonBox.value = JSON.stringify(lastInterpretedAction, null, 2);
}
showCorrectionControls(true);
recordingStatus.textContent = "Correction mode: edit the JSON above, click Preview to test drawing on the canvas. Only Accept when the sample looks right.";
// Trigger a validation run
jsonBox.dispatchEvent(new Event('input'));
}
function previewCorrection() {
if (!jsonBox || !jsonBox.value || !correctionSnapshot) {
recordingStatus.textContent = "No correction snapshot or JSON to preview.";
return;
}
try {
const editedAction = JSON.parse(jsonBox.value);
if (!Array.isArray(editedAction)) throw new Error("JSON must be a plain array of actions — remove any 'canonical' wrapper if present");
// Always base the preview off the original snapshot (so repeated previews are clean)
shapes = JSON.parse(JSON.stringify(correctionSnapshot));
// Apply the candidate corrected actions (same logic as before)
editedAction.forEach(action => {
if (action.shape === 'clear') {
shapes = [];
} else {
shapes.push(action);
}
});
// Update runtime state so Accept knows what we approved
lastInterpretedAction = editedAction;
renderAllShapes();
if (confirmCorrectionBtn) confirmCorrectionBtn.disabled = false;
recordingStatus.textContent = "Sample execution shown on canvas. Review the result. Click Accept to store this mapping in MemRL with +1.0 reinforcement.";
} catch (e) {
if (jsonValidationEl) {
jsonValidationEl.textContent = '✗ ' + e.message;
jsonValidationEl.className = 'json-validation bad';
} else {
alert("Invalid JSON for preview: " + e.message);
}
}
}
async function confirmCorrectionAndReinforce() {
if (!lastTranscription || !lastInterpretedAction || !correctionSnapshot) return;
// At this point the canvas already shows the previewed/good result.
// Send with +1.0 (positive reinforcement) and provide corrected_action_json
// so backend can ensure the *good* action is the one associated with the (noisy) query.
let reqData = {
query: lastTranscription,
action_json: JSON.stringify(lastInterpretedAction),
reward: 1.0,
corrected_action_json: JSON.stringify(lastInterpretedAction)
};
try {
const response = await fetch('/api/reinforce', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(reqData)
});
if (!response.ok) throw new Error("Reinforce API update failed");
const result = await response.json();
recordingStatus.textContent = result.message || "Correction accepted with positive reinforcement.";
loadMemories();
rewardCard.classList.add('disabled');
// Clean up correction mode (keep the good drawing on canvas)
correctionSnapshot = null;
showCorrectionControls(false);
} catch (err) {
console.error("Error committing corrected reinforcement:", err);
recordingStatus.textContent = "Failed to save correction to MemRL. You can try again.";
}
}
function cancelCorrection() {
if (correctionSnapshot) {
shapes = correctionSnapshot;
renderAllShapes();
}
// Restore the original interpreted action in the box if we have it
if (lastInterpretedAction && jsonBox) {
jsonBox.value = JSON.stringify(lastInterpretedAction, null, 2);
}
correctionSnapshot = null;
showCorrectionControls(false);
recordingStatus.textContent = "Correction cancelled. Canvas reverted to previous state.";
}
// ==========================================
// 8. Q-Memory Database Loader
// ==========================================
const memoryTableBody = document.getElementById('memoryTableBody');
const resetMemoryBtn = document.getElementById('resetMemoryBtn');
resetMemoryBtn.addEventListener('click', wipeDatabase);
async function loadMemories() {
try {
const response = await fetch('/api/memories');
if (!response.ok) throw new Error("Failed to load DB memories");
const data = await response.json();
populateMemoryTable(data.memories);
} catch (err) {
console.error("Error reading Q-database:", err);
}
}
function populateMemoryTable(memories) {
memoryTableBody.innerHTML = '';
if (!memories || memories.length === 0) {
memoryTableBody.innerHTML = `
<tr>
<td colspan="3" class="empty-state">No learned memories yet. Use voice activation to begin.</td>
</tr>
`;
return;
}
memories.forEach(mem => {
const row = document.createElement('tr');
let qClass = 'q-low';
if (mem.q_value >= 0.8) qClass = 'q-high';
else if (mem.q_value >= 0.4) qClass = 'q-mid';
const actionStr = JSON.stringify(mem.action);
const lang = mem.language || 'multi';
const canon = mem.canonical_intent ? ` <span style="color:#00e699;font-size:0.7em">[${mem.canonical_intent}]</span>` : '';
row.innerHTML = `
<td>"${mem.query}" <small style="opacity:0.6">(${lang})</small>${canon}</td>
<td><span class="action-badge" title='${actionStr}'>${actionStr}</span></td>
<td class="q-value-cell ${qClass}">${mem.q_value.toFixed(3)}</td>
`;
memoryTableBody.appendChild(row);
});
}
async function wipeDatabase() {
if (!confirm("Are you sure you want to reset all Q-values? This deletes all custom mapping history.")) return;
try {
const response = await fetch('/api/clear_memories', { method: 'POST' });
if (!response.ok) throw new Error("Wipe failed");
shapes = [];
renderAllShapes();
lastTranscription = "";
lastInterpretedAction = [];
transcriptionBox.textContent = "None";
jsonBox.value = "";
updateBadge("IDLE", "state-idle");
rewardCard.classList.add('disabled');
resetCorrectionState();
recordingStatus.textContent = "Database reset complete.";
loadMemories();
} catch (err) {
console.error("Error clearing SQLite DB:", err);
}
}
// ==========================================
// 9. Manual Palette & Settings Callbacks
// ==========================================
const toolBtns = document.querySelectorAll('.tool-btn');
const colorBtns = document.querySelectorAll('.color-btn');
const sizeSlider = document.getElementById('sizeSlider');
const sizeValText = document.getElementById('sizeVal');
const customColorPicker = document.getElementById('customColorPicker');
const manualClearBtn = document.getElementById('clearBtn');
const gridToggle = document.getElementById('gridToggle');
const animateToggle = document.getElementById('animateToggle');
// Dynamic listeners
gridToggle.addEventListener('change', renderAllShapes);
animateToggle.addEventListener('change', renderAllShapes);
toolBtns.forEach(btn => {
btn.addEventListener('click', () => {
toolBtns.forEach(b => b.classList.remove('active'));
btn.classList.add('active');
currentTool = btn.dataset.tool;
});
});
colorBtns.forEach(btn => {
btn.addEventListener('click', () => {
colorBtns.forEach(b => b.classList.remove('active'));
btn.classList.add('active');
currentColor = btn.dataset.color;
customColorPicker.value = translateColorToHex(currentColor);
});
});
customColorPicker.addEventListener('input', (e) => {
colorBtns.forEach(b => b.classList.remove('active'));
currentColor = e.target.value;
});
sizeSlider.addEventListener('input', (e) => {
strokeSize = parseInt(e.target.value);
sizeValText.textContent = `${strokeSize}px`;
});
manualClearBtn.addEventListener('click', clearCanvas);
function translateColorToHex(cName) {
const hexMap = {
red: '#ff0000',
blue: '#0000ff',
green: '#008000',
yellow: '#ffff00',
purple: '#800080',
orange: '#ffa500',
black: '#000000'
};
return hexMap[cName] || '#000000';
}
// Manual drawing at click point
canvas.addEventListener('click', (event) => {
if (draggedShape) return; // avoid drawing when finishing a drag
const rect = canvas.getBoundingClientRect();
const x = event.clientX - rect.left;
const y = event.clientY - rect.top;
// Renders shape centered at click point
let clickAction = {
shape: currentTool,
color: currentColor,
size: strokeSize,
x: x,
y: y
};
if (currentTool === 'text') {
const label = window.prompt('Text label?', 'label') || 'label';
clickAction.text = label;
clickAction.size = Math.max(24, Math.min(strokeSize, 120));
}
executeCanvasActions([clickAction]);
});
// Setup click handlers for example pills
const examplePills = document.querySelectorAll('.example-cmd-pill');
examplePills.forEach(pill => {
pill.addEventListener('click', () => {
const text = pill.textContent.replace(/"/g, '').trim();
// Emulate receiving this text from speech
runMockTextCommand(text);
});
});
async function runMockTextCommand(text) {
recordingStatus.textContent = "Processing mock command...";
transcriptionBox.textContent = `"${text}"`;
updateBadge("PROCESSING", "state-suggest");
// Demo path for example pills: client-side memory lookup (levenshtein) + local rule fallback.
// This is fast, fully offline, and lets users test + immediately reinforce via the feedback UI.
// (We fire a dummy to /api/transcribe only for side-effect consistency; result is ignored.)
try {
// Fire-and-forget dummy to keep API path warm (optional)
fetch(`/api/transcribe`, {
method: 'POST',
body: createFormWithDummyAudio()
}).catch(() => {});
lastTranscription = text;
// Query memory locally by sending query? No, let's fetch memories list and look up:
const memResponse = await fetch('/api/memories');
const memData = await memResponse.json();
let match = null;
let max_sim = 0.0;
// Custom js levenshtein logic
const threshold = 0.75;
memData.memories.forEach(mem => {
const sim = levenshtein_similarity(text, mem.query);
if (sim > max_sim) {
max_sim = sim;
match = mem;
}
});
if (max_sim >= threshold && match) {
resetCorrectionState();
lastInterpretedAction = match.action;
jsonBox.value = JSON.stringify(lastInterpretedAction, null, 2);
rewardCard.classList.remove('disabled');
if (match.q_value >= 0.8) {
updateBadge("MEMRL RECALL (1.0)", "state-matched");
executeCanvasActions(lastInterpretedAction);
recordingStatus.textContent = "Auto-executed from memory!";
} else {
updateBadge("SUGGESTION MAP", "state-suggest");
showSuggestionBanner(text, lastInterpretedAction);
recordingStatus.textContent = "Suggestion found in memory.";
}
} else {
// Fallback rule based parse locally to render instantly!
// This ensures instant demo feedback for pills.
resetCorrectionState();
const localFallback = localRuleBasedFallback(text);
lastInterpretedAction = localFallback;
jsonBox.value = JSON.stringify(lastInterpretedAction, null, 2);
rewardCard.classList.remove('disabled');
updateBadge("LOCAL INTERPRETER", "state-miss");
executeCanvasActions(lastInterpretedAction);
recordingStatus.textContent = "Resolved via local parser.";
}
} catch (e) {
console.error("Mock pill run failed:", e);
}
}
function createFormWithDummyAudio() {
const dummyBlob = new Blob([new Uint8Array(100)], { type: 'audio/wav' });
const fd = new FormData();
fd.append('audio', dummyBlob, 'dummy.wav');
return fd;
}
function levenshtein_similarity(s1, s2) {
// Match Python implementation in app.py exactly for symmetry (per Agents.md):
// lower + strip whitespace + strip punctuation chars from ends.
s1 = (s1 || "").toLowerCase().trim().replace(/^[.?,!]+|[.?,!]+$/g, '');
s2 = (s2 || "").toLowerCase().trim().replace(/^[.?,!]+|[.?,!]+$/g, '');
if (s1 === s2) return 1.0;
const m = s1.length, n = s2.length;
if (Math.max(m, n) === 0) return 1.0;
const dp = Array(m + 1).fill().map(() => Array(n + 1).fill(0));
for (let i = 0; i <= m; i++) dp[i][0] = i;
for (let j = 0; j <= n; j++) dp[0][j] = j;
for (let i = 1; i <= m; i++) {
for (let j = 1; j <= n; j++) {
if (s1[i-1] === s2[j-1]) dp[i][j] = dp[i-1][j-1];
else dp[i][j] = Math.min(dp[i-1][j], dp[i][j-1], dp[i-1][j-1]) + 1;
}
}
return 1.0 - (dp[m][n] / Math.max(m, n));
}
// Client-side rule based fallback matching localRuleBasedFallback
function localRuleBasedFallback(user_text) {
const lower_text = user_text.toLowerCase();
const parts = lower_text.split(/\band\b|\bthen\b|,/);
const actions = [];
const shapes_list = ["circle", "square", "rectangle", "triangle", "star", "pentagon", "hexagon", "oval", "heart", "line", "arrow", "text"];
// Multilingual color map for demo (Spanish/French etc.)
const color_map = {
"red": ["red", "rojo", "rouge"],
"blue": ["blue", "azul", "bleu"],
"green": ["green", "verde", "vert"],
"yellow": ["yellow", "amarillo", "jaune"],
"orange": ["orange", "naranja"],
"purple": ["purple", "morado", "violet", "pourpre"],
"pink": ["pink", "rosa", "rose"],
"black": ["black", "negro", "noir"],
"white": ["white", "blanco", "blanc"],
"cyan": ["cyan"],
"magenta": ["magenta"],
};
parts.forEach(part => {
part = part.trim();
if (!part) return;
if (part.includes("clear") || part.includes("wipe") || part.includes("reset")) {
actions.push({shape: 'clear', color: 'white', size: 0});
return;
}
let color = "black";
for (let eng in color_map) {
if (color_map[eng].some(v => part.includes(v))) {
color = eng;
break;
}
}
let size = 100;
const size_match = part.match(/\b(\d+)\b/);
if (size_match) {
const val = parseInt(size_match[1]);
if (val >= 30 && val <= 300) size = val;
}
let px = "center";
if (part.includes("left")) px = "left";
else if (part.includes("right")) px = "right";
let py = "center";
if (part.includes("top")) py = "top";
else if (part.includes("bottom")) py = "bottom";
// Text label (write, say, text, label) - per Agents.md roadmap
const textMatch = part.match(/["']([^"']+)["']/);
const isTextCmd = /text|write|say|label|title/.test(part);
if (isTextCmd) {
const label = textMatch ? textMatch[1].trim() : (part.split(/\s+/).pop() || "label");
const fontSize = Math.max(24, Math.min(size, 160));
actions.push({
shape: 'text',
text: label,
color: color,
size: fontSize,
x: px,
y: py
});
return;
}
// Check for relative paths
if (part.includes("path") || part.includes("freehand") || part.includes("go") || part.includes("move")) {
const operations = [];
operations.push({type: 'start', x: px, y: py});
const words = part.split(/\s+/);
words.forEach((word, idx) => {
if (["right", "east"].includes(word) && idx + 1 < words.length) {
const val = parseInt(words[idx+1].replace(/[^0-9]/g, ''));
if (!isNaN(val)) operations.push({type: 'line', dx: val, dy: 0});
} else if (["left", "west"].includes(word) && idx + 1 < words.length) {
const val = parseInt(words[idx+1].replace(/[^0-9]/g, ''));
if (!isNaN(val)) operations.push({type: 'line', dx: -val, dy: 0});
} else if (["down", "south"].includes(word) && idx + 1 < words.length) {
const val = parseInt(words[idx+1].replace(/[^0-9]/g, ''));
if (!isNaN(val)) operations.push({type: 'line', dx: 0, dy: val});
} else if (["up", "north"].includes(word) && idx + 1 < words.length) {
const val = parseInt(words[idx+1].replace(/[^0-9]/g, ''));
if (!isNaN(val)) operations.push({type: 'line', dx: 0, dy: -val});
} else if (word === "dot" || word === "point") {
operations.push({type: 'dot', radius: 8});
}
});
if (operations.length > 1) {
actions.push({
shape: 'path',
color: color,
thickness: 5,
operations: operations
});
}
return;
}
let shape_found = null;
shapes_list.forEach(s => {
if (part.includes(s)) shape_found = s;
});
if (shape_found === "line" || shape_found === "arrow") {
const start_x = part.includes("left") ? "left" : "center";
const start_y = part.includes("top") ? "top" : "center";
const end_x = part.includes("right") ? "right" : 350;
const end_y = part.includes("bottom") ? "bottom" : 350;
actions.push({
shape: shape_found,
color: color,
start_x: start_x,
start_y: start_y,
end_x: end_x,
end_y: end_y
});
} else if (shape_found) {
actions.push({
shape: shape_found,
color: color,
size: size,
x: px,
y: py
});
} else {
actions.push({
shape: 'circle',
color: color,
size: size,
x: px,
y: py
});
}
});
return actions;
}
// Initialize on load
window.addEventListener('DOMContentLoaded', () => {
initCanvas();
loadMemories();
});