import { AutoModel, AutoProcessor, RawImage } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.8.1';
// DOM Elements
const video = document.getElementById('video');
const canvas = document.getElementById('canvas');
const ctx = canvas.getContext('2d');
const startBtn = document.getElementById('start-btn');
const btnIcon = document.getElementById('btn-icon');
const btnText = document.getElementById('btn-text');
const modelSelect = document.getElementById('model-select');
const toggleDetect = document.getElementById('toggle-detect');
const togglePose = document.getElementById('toggle-pose');
const thresholdInput = document.getElementById('threshold');
const thresholdValueEl = document.getElementById('threshold-value');
const fpsEl = document.getElementById('fps');
const loader = document.getElementById('loader');
const loaderText = document.getElementById('loader-text');
const statusDot = document.getElementById('status-dot');
const statusText = document.getElementById('status-text');
// State
let detectModel = null;
let poseModel = null;
let processor = null;
let isRunning = false;
let isProcessing = false;
let threshold = 0.5;
let enableDetect = true;
let enablePose = true;
let animationId = null;
// Offscreen canvas for frame capture
const offscreen = document.createElement('canvas');
const offscreenCtx = offscreen.getContext('2d');
// Constants
const COLORS = ['#6366f1', '#ec4899', '#14b8a6', '#f59e0b', '#8b5cf6', '#ef4444', '#10b981', '#3b82f6'];
const SKELETON = [
[0, 1], [0, 2], [1, 3], [2, 4],
[5, 6], [5, 7], [7, 9], [6, 8], [8, 10],
[5, 11], [6, 12], [11, 12],
[11, 13], [13, 15], [12, 14], [14, 16]
];
const POSE_THRESHOLD = 0.0001;
// UI Helpers
const setStatus = (text, type = 'default') => {
statusText.textContent = text;
statusDot.className = 'status-dot ' + type;
};
const showLoader = (text) => {
loaderText.textContent = text;
loader.classList.add('visible');
};
const hideLoader = () => loader.classList.remove('visible');
// Model Loading
async function loadModels(modelId) {
try {
if (isRunning) stopCamera(true);
while (isProcessing) await new Promise(r => setTimeout(r, 50));
if (detectModel) await detectModel.dispose();
if (poseModel) await poseModel.dispose();
detectModel = poseModel = processor = null;
startBtn.disabled = true;
const poseModelId = modelId.replace('-ONNX', '-pose-ONNX');
const progressCallback = (label) => (info) => {
if (info.status === 'progress' && info.file.endsWith('.onnx')) {
showLoader(`${label} (${Math.round((info.loaded / info.total) * 100)}%)`);
}
};
setStatus('Loading...', 'loading');
showLoader('Loading detection model...');
detectModel = await AutoModel.from_pretrained(modelId, {
device: 'webgpu',
dtype: 'fp16',
progress_callback: progressCallback('Loading detection model')
});
showLoader('Loading pose model...');
poseModel = await AutoModel.from_pretrained(poseModelId, {
device: 'webgpu',
dtype: 'fp32',
progress_callback: progressCallback('Loading pose model')
});
showLoader('Loading processor...');
processor = await AutoProcessor.from_pretrained(modelId);
setStatus('Ready', 'ready');
hideLoader();
startBtn.disabled = false;
startCamera();
} catch (error) {
console.error('Model loading failed:', error);
setStatus('Error', 'error');
showLoader('Failed: ' + error.message);
}
}
// Camera Control
async function startCamera() {
try {
showLoader('Accessing camera...');
const stream = await navigator.mediaDevices.getUserMedia({
video: { facingMode: 'environment', width: { ideal: 640 }, height: { ideal: 640 } },
audio: false
});
video.srcObject = stream;
video.onloadedmetadata = () => {
canvas.width = offscreen.width = video.videoWidth;
canvas.height = offscreen.height = video.videoHeight;
isRunning = true;
btnIcon.innerHTML = '';
btnText.textContent = 'Stop Camera';
startBtn.classList.add('running');
hideLoader();
setStatus('Running', 'running');
loop();
};
} catch (error) {
console.error('Camera error:', error);
setStatus('Camera Error', 'error');
showLoader('Camera access denied');
}
}
function stopCamera(keepProcessingFlag = false) {
if (animationId) cancelAnimationFrame(animationId);
animationId = null;
if (video.srcObject) {
video.srcObject.getTracks().forEach(t => t.stop());
video.srcObject = null;
}
isRunning = false;
if (!keepProcessingFlag) isProcessing = false;
btnIcon.innerHTML = '';
btnText.textContent = 'Start Camera';
startBtn.classList.remove('running');
canvas.width = canvas.width; // Clear canvas
setStatus('Ready', 'ready');
fpsEl.textContent = '0';
}
// Detection Loop
function loop() {
if (!isRunning) return;
if (detectModel && poseModel && processor && !isProcessing) {
isProcessing = true;
const startTime = performance.now();
detect()
.then(() => fpsEl.textContent = Math.round(1000 / (performance.now() - startTime)))
.finally(() => isProcessing = false);
}
if (isRunning) animationId = requestAnimationFrame(loop);
}
async function detect() {
offscreenCtx.drawImage(video, 0, 0);
const image = RawImage.fromCanvas(offscreen);
const inputs = await processor(image);
const promises = [];
if (enableDetect) promises.push(detectModel(inputs));
if (enablePose) promises.push(poseModel(inputs));
const results = await Promise.all(promises);
let idx = 0;
const detectOutput = enableDetect ? results[idx++] : null;
const poseOutput = enablePose ? results[idx++] : null;
const detections = [];
if (detectOutput) {
const scores = detectOutput.logits.sigmoid().data;
const boxes = detectOutput.pred_boxes.data;
const id2label = detectModel.config.id2label;
for (let i = 0; i < 300; i++) {
let maxScore = 0, maxClass = 0;
for (let j = 0; j < 80; j++) {
const score = scores[i * 80 + j];
if (score > maxScore) { maxScore = score; maxClass = j; }
}
if (maxScore >= threshold) {
const [cx, cy, w, h] = [boxes[i * 4], boxes[i * 4 + 1], boxes[i * 4 + 2], boxes[i * 4 + 3]];
detections.push({
type: 'object',
box: [(cx - w / 2) * canvas.width, (cy - h / 2) * canvas.height, w * canvas.width, h * canvas.height],
score: maxScore,
classId: maxClass,
label: id2label[maxClass] || `Class ${maxClass}`
});
}
}
}
if (poseOutput) {
const data = Object.values(poseOutput)[0].data;
for (let i = 0; i < 300; i++) {
const offset = i * 57;
const score = data[offset + 4];
if (score >= threshold) {
const keypoints = [];
for (let k = 0; k < 17; k++) {
const kIdx = offset + 6 + k * 3;
keypoints.push({ x: data[kIdx] * canvas.width, y: data[kIdx + 1] * canvas.height, c: data[kIdx + 2] });
}
detections.push({
type: 'pose',
box: [data[offset] * canvas.width, data[offset + 1] * canvas.height,
(data[offset + 2] - data[offset]) * canvas.width, (data[offset + 3] - data[offset + 1]) * canvas.height],
score,
keypoints
});
}
}
}
if (isRunning) draw(detections);
}
// Drawing
function draw(detections) {
ctx.clearRect(0, 0, canvas.width, canvas.height);
for (const det of detections.filter(d => d.type === 'object')) {
const [x, y, w, h] = det.box;
const color = COLORS[det.classId % COLORS.length];
const label = `${det.label} ${Math.round(det.score * 100)}%`;
ctx.strokeStyle = color;
ctx.lineWidth = 2;
ctx.strokeRect(x, y, w, h);
ctx.font = 'bold 12px system-ui';
const tw = ctx.measureText(label).width;
ctx.fillStyle = color;
ctx.fillRect(x, y > 18 ? y - 18 : y, tw + 8, 18);
ctx.fillStyle = '#fff';
ctx.fillText(label, x + 4, y > 18 ? y - 5 : y + 13);
}
for (const det of detections.filter(d => d.type === 'pose')) {
ctx.lineWidth = 3;
ctx.strokeStyle = '#22d3ee';
for (const [i, j] of SKELETON) {
const a = det.keypoints[i], b = det.keypoints[j];
if (a?.c >= POSE_THRESHOLD && b?.c >= POSE_THRESHOLD) {
ctx.beginPath();
ctx.moveTo(a.x, a.y);
ctx.lineTo(b.x, b.y);
ctx.stroke();
}
}
for (const kp of det.keypoints) {
if (kp.c < POSE_THRESHOLD) continue;
ctx.fillStyle = '#6366f1';
ctx.beginPath();
ctx.arc(kp.x, kp.y, 5, 0, Math.PI * 2);
ctx.fill();
ctx.strokeStyle = '#fff';
ctx.lineWidth = 2;
ctx.stroke();
}
}
}
// Event Listeners
startBtn.addEventListener('click', () => isRunning ? stopCamera() : startCamera());
thresholdInput.addEventListener('input', (e) => {
threshold = e.target.value / 100;
thresholdValueEl.textContent = `${e.target.value}%`;
});
toggleDetect.addEventListener('change', (e) => enableDetect = e.target.checked);
togglePose.addEventListener('change', (e) => enablePose = e.target.checked);
modelSelect.addEventListener('change', (e) => loadModels(e.target.value));
// Initialize
loadModels(modelSelect.value);