document.addEventListener('DOMContentLoaded', async () => {
// Initialize elements
const videoElem = document.getElementById('video');
const overlay = document.getElementById('overlay');
const ctx = overlay.getContext('2d');
const status = document.getElementById('status');
const predictionSpan = document.getElementById('prediction');
const confidenceBar = document.getElementById('confidenceBar');
const datasetJSON = document.getElementById('datasetJSON');
let labels = [];
let recording = false;
let recordingLabel = null;
let predicting = false;
let knnK = 5;
let lastSpoken = null;
// Set initial status
setStatus('Initializing camera...');
// === Utility Functions ===
function setStatus(s) {
status.textContent = `Status: ${s}`;
}
function speak(text) {
if ('speechSynthesis' in window && text !== lastSpoken) {
lastSpoken = text;
const msg = new SpeechSynthesisUtterance(text);
msg.rate = 1;
msg.pitch = 1;
speechSynthesis.cancel();
speechSynthesis.speak(msg);
}
}
// === Landmark Processing ===
function landmarksToVector(landmarks) {
if (!landmarks || landmarks.length === 0) return null;
const wrist = landmarks[0];
const ref = landmarks[9] || landmarks[8];
const dx = ref.x - wrist.x;
const dy = ref.y - wrist.y;
const scale = Math.hypot(dx, dy) || 1;
const vec = [];
for (let p of landmarks) {
vec.push(
(p.x - wrist.x) / scale,
(p.y - wrist.y) / scale,
(p.z - wrist.z) / scale
);
}
return vec;
}
function euclid(a, b) {
let s = 0;
for (let i = 0; i < a.length; i++) {
const d = a[i] - b[i];
s += d * d;
}
return Math.sqrt(s);
}
function knnPredict(vec) {
const all = [];
for (const lab of labels) {
for (const ex of lab.examples) {
all.push({ label: lab.name, dist: euclid(vec, ex) });
}
}
if (!all.length) return { label: null, confidence: 0 };
all.sort((a, b) => a.dist - b.dist);
const k = Math.min(knnK, all.length);
const top = all.slice(0, k);
const counts = {};
for (const t of top) counts[t.label] = (counts[t.label] || 0) + 1;
let best = null, bestCount = 0;
for (const l in counts) {
if (counts[l] > bestCount) {
best = l;
bestCount = counts[l];
}
}
return {
label: best,
confidence: bestCount / k
};
}
// === UI: Labels Management ===
document.getElementById('addLabel').addEventListener('click', () => {
const name = document.getElementById('labelInput').value.trim();
if (!name) return alert('Please enter a label name');
if (labels.find(l => l.name === name)) return alert('Label already exists');
labels.push({ name, examples: [] });
renderLabels();
document.getElementById('labelInput').value = '';
});
function renderLabels() {
const labelsArea = document.getElementById('labelsArea');
labelsArea.innerHTML = '';
labels.forEach((lab, idx) => {
const div = document.createElement('div');
div.className = 'flex items-center justify-between bg-gray-700 p-2 rounded-lg';
const labelBtn = document.createElement('button');
labelBtn.className = `flex items-center gap-2 px-3 py-1 rounded-md transition ${
recordingLabel === lab.name ? 'bg-teal-600 text-white' : 'bg-gray-600 hover:bg-gray-500'
}`;
labelBtn.innerHTML = `
${lab.name} (${lab.examples.length})
`;
labelBtn.addEventListener('click', () => {
recordingLabel = lab.name;
renderLabels();
});
const deleteBtn = document.createElement('button');
deleteBtn.className = 'p-1 text-red-400 hover:text-red-300 rounded-full';
deleteBtn.innerHTML = '';
deleteBtn.addEventListener('click', () => {
if (confirm(`Delete label "${lab.name}"?`)) {
labels.splice(idx, 1);
if (recordingLabel === lab.name) recordingLabel = null;
renderLabels();
}
});
div.appendChild(labelBtn);
div.appendChild(deleteBtn);
labelsArea.appendChild(div);
});
datasetJSON.value = JSON.stringify(labels, null, 2);
feather.replace();
}
// === Data Management ===
document.getElementById('saveBtn').addEventListener('click', () => {
if (labels.length === 0) return alert('No data to save');
const blob = new Blob([JSON.stringify(labels)], { type: 'application/json' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'handspeak_dataset.json';
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
URL.revokeObjectURL(url);
setStatus('Dataset saved');
});
document.getElementById('loadBtn').addEventListener('click', () => {
document.getElementById('fileInput').click();
});
document.getElementById('fileInput').addEventListener('change', (e) => {
const file = e.target.files[0];
if (!file) return;
const reader = new FileReader();
reader.onload = () => {
try {
const data = JSON.parse(reader.result);
if (!Array.isArray(data)) throw new Error('Invalid format');
labels = data.map(l => ({
name: l.name,
examples: l.examples || []
}));
renderLabels();
setStatus('Dataset loaded successfully');
} catch (err) {
alert('Error loading file: ' + err.message);
}
};
reader.readAsText(file);
});
// === Recording & Prediction ===
document.getElementById('toggleRec').addEventListener('click', () => {
recording = !recording;
if (recording && !recordingLabel) {
alert('Please select a label first');
recording = false;
return;
}
setStatus(recording ? `Recording examples for "${recordingLabel}"` : 'Ready');
document.getElementById('toggleRec').textContent = recording ?
'Stop Recording' : 'Start Recording Examples';
document.getElementById('toggleRec').className = recording ?
'w-full bg-red-600 hover:bg-red-500 text-white px-4 py-3 rounded-lg font-medium transition' :
'w-full bg-blue-600 hover:bg-blue-500 text-white px-4 py-3 rounded-lg font-medium transition';
});
document.getElementById('predictToggle').addEventListener('click', () => {
predicting = !predicting;
if (predicting && labels.length === 0) {
alert('Please add some labels and examples first');
predicting = false;
return;
}
setStatus(predicting ? 'Predicting...' : 'Ready');
document.getElementById('predictToggle').textContent = predicting ?
'Stop Predicting' : 'Start Predicting';
document.getElementById('predictToggle').className = predicting ?
'w-full bg-purple-700 hover:bg-purple-600 text-white px-4 py-3 rounded-lg font-medium transition' :
'w-full bg-purple-600 hover:bg-purple-500 text-white px-4 py-3 rounded-lg font-medium transition';
});
// === MediaPipe Hands Setup ===
const hands = new Hands({
locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/hands/${file}`
});
hands.setOptions({
maxNumHands: 1,
modelComplexity: 1,
minDetectionConfidence: 0.7,
minTrackingConfidence: 0.6
});
hands.onResults(onResults);
const camera = new Camera(videoElem, {
onFrame: async () => {
await hands.send({ image: videoElem });
},
width: 640,
height: 480
});
camera.start();
// Results Handler
function onResults(results) {
ctx.save();
ctx.clearRect(0, 0, overlay.width, overlay.height);
ctx.drawImage(results.image, 0, 0, overlay.width, overlay.height);
if (results.multiHandLandmarks) {
for (const landmarks of results.multiHandLandmarks) {
drawConnectors(ctx, landmarks, HAND_CONNECTIONS, {
color: '#10B981',
lineWidth: 4
});
drawLandmarks(ctx, landmarks, {
color: '#F59E0B',
lineWidth: 2,
radius: (data) => {
return lerp(data.from.z, -0.15, 0.1, 5, 1);
}
});
}
}
ctx.restore();
// Process landmarks for recording/prediction