|
|
document.addEventListener('DOMContentLoaded', async () => { |
|
|
|
|
|
const videoElem = document.getElementById('video'); |
|
|
const overlay = document.getElementById('overlay'); |
|
|
const ctx = overlay.getContext('2d'); |
|
|
const status = document.getElementById('status'); |
|
|
const predictionSpan = document.getElementById('prediction'); |
|
|
const confidenceBar = document.getElementById('confidenceBar'); |
|
|
const datasetJSON = document.getElementById('datasetJSON'); |
|
|
|
|
|
let labels = []; |
|
|
let recording = false; |
|
|
let recordingLabel = null; |
|
|
let predicting = false; |
|
|
let knnK = 5; |
|
|
let lastSpoken = null; |
|
|
|
|
|
|
|
|
setStatus('Initializing camera...'); |
|
|
|
|
|
|
|
|
function setStatus(s) { |
|
|
status.textContent = `Status: ${s}`; |
|
|
} |
|
|
|
|
|
function speak(text) { |
|
|
if ('speechSynthesis' in window && text !== lastSpoken) { |
|
|
lastSpoken = text; |
|
|
const msg = new SpeechSynthesisUtterance(text); |
|
|
msg.rate = 1; |
|
|
msg.pitch = 1; |
|
|
speechSynthesis.cancel(); |
|
|
speechSynthesis.speak(msg); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
function landmarksToVector(landmarks) { |
|
|
if (!landmarks || landmarks.length === 0) return null; |
|
|
|
|
|
const wrist = landmarks[0]; |
|
|
const ref = landmarks[9] || landmarks[8]; |
|
|
const dx = ref.x - wrist.x; |
|
|
const dy = ref.y - wrist.y; |
|
|
const scale = Math.hypot(dx, dy) || 1; |
|
|
|
|
|
const vec = []; |
|
|
for (let p of landmarks) { |
|
|
vec.push( |
|
|
(p.x - wrist.x) / scale, |
|
|
(p.y - wrist.y) / scale, |
|
|
(p.z - wrist.z) / scale |
|
|
); |
|
|
} |
|
|
return vec; |
|
|
} |
|
|
|
|
|
function euclid(a, b) { |
|
|
let s = 0; |
|
|
for (let i = 0; i < a.length; i++) { |
|
|
const d = a[i] - b[i]; |
|
|
s += d * d; |
|
|
} |
|
|
return Math.sqrt(s); |
|
|
} |
|
|
|
|
|
function knnPredict(vec) { |
|
|
const all = []; |
|
|
for (const lab of labels) { |
|
|
for (const ex of lab.examples) { |
|
|
all.push({ label: lab.name, dist: euclid(vec, ex) }); |
|
|
} |
|
|
} |
|
|
|
|
|
if (!all.length) return { label: null, confidence: 0 }; |
|
|
|
|
|
all.sort((a, b) => a.dist - b.dist); |
|
|
const k = Math.min(knnK, all.length); |
|
|
const top = all.slice(0, k); |
|
|
|
|
|
const counts = {}; |
|
|
for (const t of top) counts[t.label] = (counts[t.label] || 0) + 1; |
|
|
|
|
|
let best = null, bestCount = 0; |
|
|
for (const l in counts) { |
|
|
if (counts[l] > bestCount) { |
|
|
best = l; |
|
|
bestCount = counts[l]; |
|
|
} |
|
|
} |
|
|
|
|
|
return { |
|
|
label: best, |
|
|
confidence: bestCount / k |
|
|
}; |
|
|
} |
|
|
|
|
|
|
|
|
document.getElementById('addLabel').addEventListener('click', () => { |
|
|
const name = document.getElementById('labelInput').value.trim(); |
|
|
if (!name) return alert('Please enter a label name'); |
|
|
if (labels.find(l => l.name === name)) return alert('Label already exists'); |
|
|
|
|
|
labels.push({ name, examples: [] }); |
|
|
renderLabels(); |
|
|
document.getElementById('labelInput').value = ''; |
|
|
}); |
|
|
|
|
|
function renderLabels() { |
|
|
const labelsArea = document.getElementById('labelsArea'); |
|
|
labelsArea.innerHTML = ''; |
|
|
|
|
|
labels.forEach((lab, idx) => { |
|
|
const div = document.createElement('div'); |
|
|
div.className = 'flex items-center justify-between bg-gray-700 p-2 rounded-lg'; |
|
|
|
|
|
const labelBtn = document.createElement('button'); |
|
|
labelBtn.className = `flex items-center gap-2 px-3 py-1 rounded-md transition ${ |
|
|
recordingLabel === lab.name ? 'bg-teal-600 text-white' : 'bg-gray-600 hover:bg-gray-500' |
|
|
}`; |
|
|
labelBtn.innerHTML = ` |
|
|
<i data-feather="${lab.name.length > 1 ? 'type' : 'hash'}" width="16"></i> |
|
|
${lab.name} <span class="text-xs opacity-75">(${lab.examples.length})</span> |
|
|
`; |
|
|
labelBtn.addEventListener('click', () => { |
|
|
recordingLabel = lab.name; |
|
|
renderLabels(); |
|
|
}); |
|
|
|
|
|
const deleteBtn = document.createElement('button'); |
|
|
deleteBtn.className = 'p-1 text-red-400 hover:text-red-300 rounded-full'; |
|
|
deleteBtn.innerHTML = '<i data-feather="trash-2" width="16"></i>'; |
|
|
deleteBtn.addEventListener('click', () => { |
|
|
if (confirm(`Delete label "${lab.name}"?`)) { |
|
|
labels.splice(idx, 1); |
|
|
if (recordingLabel === lab.name) recordingLabel = null; |
|
|
renderLabels(); |
|
|
} |
|
|
}); |
|
|
|
|
|
div.appendChild(labelBtn); |
|
|
div.appendChild(deleteBtn); |
|
|
labelsArea.appendChild(div); |
|
|
}); |
|
|
|
|
|
datasetJSON.value = JSON.stringify(labels, null, 2); |
|
|
feather.replace(); |
|
|
} |
|
|
|
|
|
|
|
|
document.getElementById('saveBtn').addEventListener('click', () => { |
|
|
if (labels.length === 0) return alert('No data to save'); |
|
|
|
|
|
const blob = new Blob([JSON.stringify(labels)], { type: 'application/json' }); |
|
|
const url = URL.createObjectURL(blob); |
|
|
|
|
|
const a = document.createElement('a'); |
|
|
a.href = url; |
|
|
a.download = 'handspeak_dataset.json'; |
|
|
document.body.appendChild(a); |
|
|
a.click(); |
|
|
document.body.removeChild(a); |
|
|
URL.revokeObjectURL(url); |
|
|
|
|
|
setStatus('Dataset saved'); |
|
|
}); |
|
|
|
|
|
document.getElementById('loadBtn').addEventListener('click', () => { |
|
|
document.getElementById('fileInput').click(); |
|
|
}); |
|
|
|
|
|
document.getElementById('fileInput').addEventListener('change', (e) => { |
|
|
const file = e.target.files[0]; |
|
|
if (!file) return; |
|
|
|
|
|
const reader = new FileReader(); |
|
|
reader.onload = () => { |
|
|
try { |
|
|
const data = JSON.parse(reader.result); |
|
|
if (!Array.isArray(data)) throw new Error('Invalid format'); |
|
|
|
|
|
labels = data.map(l => ({ |
|
|
name: l.name, |
|
|
examples: l.examples || [] |
|
|
})); |
|
|
|
|
|
renderLabels(); |
|
|
setStatus('Dataset loaded successfully'); |
|
|
} catch (err) { |
|
|
alert('Error loading file: ' + err.message); |
|
|
} |
|
|
}; |
|
|
reader.readAsText(file); |
|
|
}); |
|
|
|
|
|
|
|
|
document.getElementById('toggleRec').addEventListener('click', () => { |
|
|
recording = !recording; |
|
|
|
|
|
if (recording && !recordingLabel) { |
|
|
alert('Please select a label first'); |
|
|
recording = false; |
|
|
return; |
|
|
} |
|
|
|
|
|
setStatus(recording ? `Recording examples for "${recordingLabel}"` : 'Ready'); |
|
|
document.getElementById('toggleRec').textContent = recording ? |
|
|
'Stop Recording' : 'Start Recording Examples'; |
|
|
|
|
|
document.getElementById('toggleRec').className = recording ? |
|
|
'w-full bg-red-600 hover:bg-red-500 text-white px-4 py-3 rounded-lg font-medium transition' : |
|
|
'w-full bg-blue-600 hover:bg-blue-500 text-white px-4 py-3 rounded-lg font-medium transition'; |
|
|
}); |
|
|
|
|
|
document.getElementById('predictToggle').addEventListener('click', () => { |
|
|
predicting = !predicting; |
|
|
|
|
|
if (predicting && labels.length === 0) { |
|
|
alert('Please add some labels and examples first'); |
|
|
predicting = false; |
|
|
return; |
|
|
} |
|
|
|
|
|
setStatus(predicting ? 'Predicting...' : 'Ready'); |
|
|
document.getElementById('predictToggle').textContent = predicting ? |
|
|
'Stop Predicting' : 'Start Predicting'; |
|
|
|
|
|
document.getElementById('predictToggle').className = predicting ? |
|
|
'w-full bg-purple-700 hover:bg-purple-600 text-white px-4 py-3 rounded-lg font-medium transition' : |
|
|
'w-full bg-purple-600 hover:bg-purple-500 text-white px-4 py-3 rounded-lg font-medium transition'; |
|
|
}); |
|
|
|
|
|
|
|
|
const hands = new Hands({ |
|
|
locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/hands/${file}` |
|
|
}); |
|
|
|
|
|
hands.setOptions({ |
|
|
maxNumHands: 1, |
|
|
modelComplexity: 1, |
|
|
minDetectionConfidence: 0.7, |
|
|
minTrackingConfidence: 0.6 |
|
|
}); |
|
|
|
|
|
hands.onResults(onResults); |
|
|
|
|
|
const camera = new Camera(videoElem, { |
|
|
onFrame: async () => { |
|
|
await hands.send({ image: videoElem }); |
|
|
}, |
|
|
width: 640, |
|
|
height: 480 |
|
|
}); |
|
|
|
|
|
camera.start(); |
|
|
|
|
|
|
|
|
function onResults(results) { |
|
|
ctx.save(); |
|
|
ctx.clearRect(0, 0, overlay.width, overlay.height); |
|
|
ctx.drawImage(results.image, 0, 0, overlay.width, overlay.height); |
|
|
|
|
|
if (results.multiHandLandmarks) { |
|
|
for (const landmarks of results.multiHandLandmarks) { |
|
|
drawConnectors(ctx, landmarks, HAND_CONNECTIONS, { |
|
|
color: '#10B981', |
|
|
lineWidth: 4 |
|
|
}); |
|
|
|
|
|
drawLandmarks(ctx, landmarks, { |
|
|
color: '#F59E0B', |
|
|
lineWidth: 2, |
|
|
radius: (data) => { |
|
|
return lerp(data.from.z, -0.15, 0.1, 5, 1); |
|
|
} |
|
|
}); |
|
|
} |
|
|
} |
|
|
|
|
|
ctx.restore(); |
|
|
|
|
|
|