document.addEventListener('DOMContentLoaded', async () => { // Initialize elements const videoElem = document.getElementById('video'); const overlay = document.getElementById('overlay'); const ctx = overlay.getContext('2d'); const status = document.getElementById('status'); const predictionSpan = document.getElementById('prediction'); const confidenceBar = document.getElementById('confidenceBar'); const datasetJSON = document.getElementById('datasetJSON'); let labels = []; let recording = false; let recordingLabel = null; let predicting = false; let knnK = 5; let lastSpoken = null; // Set initial status setStatus('Initializing camera...'); // === Utility Functions === function setStatus(s) { status.textContent = `Status: ${s}`; } function speak(text) { if ('speechSynthesis' in window && text !== lastSpoken) { lastSpoken = text; const msg = new SpeechSynthesisUtterance(text); msg.rate = 1; msg.pitch = 1; speechSynthesis.cancel(); speechSynthesis.speak(msg); } } // === Landmark Processing === function landmarksToVector(landmarks) { if (!landmarks || landmarks.length === 0) return null; const wrist = landmarks[0]; const ref = landmarks[9] || landmarks[8]; const dx = ref.x - wrist.x; const dy = ref.y - wrist.y; const scale = Math.hypot(dx, dy) || 1; const vec = []; for (let p of landmarks) { vec.push( (p.x - wrist.x) / scale, (p.y - wrist.y) / scale, (p.z - wrist.z) / scale ); } return vec; } function euclid(a, b) { let s = 0; for (let i = 0; i < a.length; i++) { const d = a[i] - b[i]; s += d * d; } return Math.sqrt(s); } function knnPredict(vec) { const all = []; for (const lab of labels) { for (const ex of lab.examples) { all.push({ label: lab.name, dist: euclid(vec, ex) }); } } if (!all.length) return { label: null, confidence: 0 }; all.sort((a, b) => a.dist - b.dist); const k = Math.min(knnK, all.length); const top = all.slice(0, k); const counts = {}; for (const t of top) counts[t.label] = (counts[t.label] || 0) + 1; let best = null, bestCount = 0; for (const l in counts) { if (counts[l] > bestCount) { best = l; bestCount = counts[l]; } } return { label: best, confidence: bestCount / k }; } // === UI: Labels Management === document.getElementById('addLabel').addEventListener('click', () => { const name = document.getElementById('labelInput').value.trim(); if (!name) return alert('Please enter a label name'); if (labels.find(l => l.name === name)) return alert('Label already exists'); labels.push({ name, examples: [] }); renderLabels(); document.getElementById('labelInput').value = ''; }); function renderLabels() { const labelsArea = document.getElementById('labelsArea'); labelsArea.innerHTML = ''; labels.forEach((lab, idx) => { const div = document.createElement('div'); div.className = 'flex items-center justify-between bg-gray-700 p-2 rounded-lg'; const labelBtn = document.createElement('button'); labelBtn.className = `flex items-center gap-2 px-3 py-1 rounded-md transition ${ recordingLabel === lab.name ? 'bg-teal-600 text-white' : 'bg-gray-600 hover:bg-gray-500' }`; labelBtn.innerHTML = ` ${lab.name} (${lab.examples.length}) `; labelBtn.addEventListener('click', () => { recordingLabel = lab.name; renderLabels(); }); const deleteBtn = document.createElement('button'); deleteBtn.className = 'p-1 text-red-400 hover:text-red-300 rounded-full'; deleteBtn.innerHTML = ''; deleteBtn.addEventListener('click', () => { if (confirm(`Delete label "${lab.name}"?`)) { labels.splice(idx, 1); if (recordingLabel === lab.name) recordingLabel = null; renderLabels(); } }); div.appendChild(labelBtn); div.appendChild(deleteBtn); labelsArea.appendChild(div); }); datasetJSON.value = JSON.stringify(labels, null, 2); feather.replace(); } // === Data Management === document.getElementById('saveBtn').addEventListener('click', () => { if (labels.length === 0) return alert('No data to save'); const blob = new Blob([JSON.stringify(labels)], { type: 'application/json' }); const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; a.download = 'handspeak_dataset.json'; document.body.appendChild(a); a.click(); document.body.removeChild(a); URL.revokeObjectURL(url); setStatus('Dataset saved'); }); document.getElementById('loadBtn').addEventListener('click', () => { document.getElementById('fileInput').click(); }); document.getElementById('fileInput').addEventListener('change', (e) => { const file = e.target.files[0]; if (!file) return; const reader = new FileReader(); reader.onload = () => { try { const data = JSON.parse(reader.result); if (!Array.isArray(data)) throw new Error('Invalid format'); labels = data.map(l => ({ name: l.name, examples: l.examples || [] })); renderLabels(); setStatus('Dataset loaded successfully'); } catch (err) { alert('Error loading file: ' + err.message); } }; reader.readAsText(file); }); // === Recording & Prediction === document.getElementById('toggleRec').addEventListener('click', () => { recording = !recording; if (recording && !recordingLabel) { alert('Please select a label first'); recording = false; return; } setStatus(recording ? `Recording examples for "${recordingLabel}"` : 'Ready'); document.getElementById('toggleRec').textContent = recording ? 'Stop Recording' : 'Start Recording Examples'; document.getElementById('toggleRec').className = recording ? 'w-full bg-red-600 hover:bg-red-500 text-white px-4 py-3 rounded-lg font-medium transition' : 'w-full bg-blue-600 hover:bg-blue-500 text-white px-4 py-3 rounded-lg font-medium transition'; }); document.getElementById('predictToggle').addEventListener('click', () => { predicting = !predicting; if (predicting && labels.length === 0) { alert('Please add some labels and examples first'); predicting = false; return; } setStatus(predicting ? 'Predicting...' : 'Ready'); document.getElementById('predictToggle').textContent = predicting ? 'Stop Predicting' : 'Start Predicting'; document.getElementById('predictToggle').className = predicting ? 'w-full bg-purple-700 hover:bg-purple-600 text-white px-4 py-3 rounded-lg font-medium transition' : 'w-full bg-purple-600 hover:bg-purple-500 text-white px-4 py-3 rounded-lg font-medium transition'; }); // === MediaPipe Hands Setup === const hands = new Hands({ locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/hands/${file}` }); hands.setOptions({ maxNumHands: 1, modelComplexity: 1, minDetectionConfidence: 0.7, minTrackingConfidence: 0.6 }); hands.onResults(onResults); const camera = new Camera(videoElem, { onFrame: async () => { await hands.send({ image: videoElem }); }, width: 640, height: 480 }); camera.start(); // Results Handler function onResults(results) { ctx.save(); ctx.clearRect(0, 0, overlay.width, overlay.height); ctx.drawImage(results.image, 0, 0, overlay.width, overlay.height); if (results.multiHandLandmarks) { for (const landmarks of results.multiHandLandmarks) { drawConnectors(ctx, landmarks, HAND_CONNECTIONS, { color: '#10B981', lineWidth: 4 }); drawLandmarks(ctx, landmarks, { color: '#F59E0B', lineWidth: 2, radius: (data) => { return lerp(data.from.z, -0.15, 0.1, 5, 1); } }); } } ctx.restore(); // Process landmarks for recording/prediction