CV_v1 / app.js
Dp1412's picture
Upload 3 files
e659e9f verified
Raw History Blame Contribute Delete
21.6 kB
// Use global ort loaded via CDN bundle
const { InferenceSession, Tensor, env } = window.ort;
// Configure WASM fallback paths to CDN (no local wasm binaries required)
env.wasm.wasmPaths = 'https://cdn.jsdelivr.net/npm/onnxruntime-web@1.20.1/dist/';
env.wasm.simd = true;
// ── COCO Class Names (80 standard classes) ───────────────────────────────────
const COCO = [
'person','bicycle','car','motorcycle','airplane','bus','train','truck','boat',
'traffic light','fire hydrant','stop sign','parking meter','bench','bird','cat',
'dog','horse','sheep','cow','elephant','bear','zebra','giraffe','backpack',
'umbrella','handbag','tie','suitcase','frisbee','skis','snowboard','sports ball',
'kite','baseball bat','baseball glove','skateboard','surfboard','tennis racket',
'bottle','wine glass','cup','fork','knife','spoon','bowl','banana','apple',
'sandwich','orange','broccoli','carrot','hot dog','pizza','donut','cake','chair',
'couch','potted plant','bed','dining table','toilet','tv','laptop','mouse',
'remote','keyboard','cell phone','microwave','oven','toaster','sink','refrigerator',
'book','clock','vase','scissors','teddy bear','hair drier','toothbrush'
];
// ── DOM References ───────────────────────────────────────────────────────────
const modelPreset = document.getElementById('model-preset');
const devicePreset = document.getElementById('device-preset');
const engineBadge = document.getElementById('engine-badge');
const badgeDot = document.getElementById('badge-dot');
const badgeText = document.getElementById('badge-text');
const loaderBox = document.getElementById('loader-box');
const loaderStatus = document.getElementById('loader-status');
const loaderPercent = document.getElementById('loader-percent');
const loaderBar = document.getElementById('loader-bar');
const confSlider = document.getElementById('conf-slider');
const confVal = document.getElementById('conf-val');
const personFilter = document.getElementById('person-filter');
const metricLatency = document.getElementById('metric-latency');
const metricFps = document.getElementById('metric-fps');
const metricCount = document.getElementById('metric-count');
const tabWebcam = document.getElementById('tab-webcam');
const tabMedia = document.getElementById('tab-media');
const displayCanvas = document.getElementById('display-canvas');
const ctx = displayCanvas.getContext('2d');
const liveVideo = document.getElementById('live-video');
const fileVideo = document.getElementById('file-video');
const fileImage = document.getElementById('file-image');
const camIdleOverlay = document.getElementById('cam-idle-overlay');
const mediaIdleOverlay = document.getElementById('media-idle-overlay');
const startCamBtn = document.getElementById('start-cam-btn');
const stopCamBtn = document.getElementById('stop-cam-btn');
const mediaFileInput = document.getElementById('media-file-input');
const mediaFileInput2 = document.getElementById('media-file-input-2');
const webcamControls = document.getElementById('webcam-controls');
const mediaControls = document.getElementById('media-controls');
const videoPlayBtn = document.getElementById('video-play-btn');
const toastEl = document.getElementById('toast');
// ── State ────────────────────────────────────────────────────────────────────
let session = null;
let activeArchitecture = 'yolo26'; // 'yolo26' (NMS-free [1,300,6]) or 'yolo11' (anchor-free [1,84,N])
let inputSize = 736; // Resolution exported from notebook
let inTensorName = '';
let outTensorName = '';
let currentMode = 'webcam'; // 'webcam' | 'media'
let isWebcamActive = false;
let isVideoActive = false;
let currentMediaType = null; // 'image' | 'video'
let animFrameId = null;
let lastTime = performance.now();
let frameCount = 0;
// Preprocessing offscreen canvas
let offscreenCanvas = new OffscreenCanvas(inputSize, inputSize);
let offscreenCtx = offscreenCanvas.getContext('2d', { willReadFrequently: true });
function updateOffscreenSize(sz) {
if (sz && sz !== inputSize) {
inputSize = sz;
offscreenCanvas = new OffscreenCanvas(inputSize, inputSize);
offscreenCtx = offscreenCanvas.getContext('2d', { willReadFrequently: true });
}
}
function showToast(msg) {
toastEl.textContent = msg;
toastEl.classList.add('show');
setTimeout(() => toastEl.classList.remove('show'), 3000);
}
// ── Model Downloader & Loader ────────────────────────────────────────────────
async function loadSelectedModel() {
const modelName = modelPreset.value;
const isWebGPU = devicePreset.value === 'webgpu';
badgeDot.className = 'badge-dot loading';
badgeText.textContent = 'Compiling...';
loaderBox.style.display = 'block';
loaderStatus.textContent = `Streaming ${modelName}...`;
loaderBar.style.width = '10%';
loaderPercent.textContent = '10%';
try {
// Path resolution: try local directory first, then ../models/
let modelPath = `./${modelName}.onnx`;
let res = await fetch(modelPath);
if (!res.ok) {
modelPath = `../models/${modelName}.onnx`;
res = await fetch(modelPath);
}
if (!res.ok) throw new Error(`HTTP ${res.status}: File not found at ${modelPath}`);
const contentLength = +res.headers.get('content-length') || 0;
const reader = res.body.getReader();
const chunks = [];
let received = 0;
while (true) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
received += value.length;
if (contentLength > 0) {
const pct = Math.round((received / contentLength) * 80);
loaderBar.style.width = `${pct}%`;
loaderPercent.textContent = `${pct}%`;
loaderStatus.textContent = `Downloaded ${(received / 1024 / 1024).toFixed(1)} MB`;
}
}
loaderStatus.textContent = 'Allocating GPU Buffers...';
loaderBar.style.width = '90%';
loaderPercent.textContent = '90%';
const modelBuffer = new Uint8Array(received);
let offset = 0;
for (const chunk of chunks) {
modelBuffer.set(chunk, offset);
offset += chunk.length;
}
// Check if WebGPU adapter is actually available
let useWebGPU = isWebGPU;
if (useWebGPU) {
try {
if (!navigator.gpu || !(await navigator.gpu.requestAdapter())) {
useWebGPU = false;
}
} catch (e) {
useWebGPU = false;
}
}
const providers = useWebGPU ? ['webgpu', 'wasm'] : ['wasm'];
try {
session = await InferenceSession.create(modelBuffer.buffer, {
executionProviders: providers,
graphOptimizationLevel: 'all'
});
} catch (createErr) {
console.warn('Primary EP init fallback to WASM:', createErr);
session = await InferenceSession.create(modelBuffer.buffer, {
executionProviders: ['wasm'],
graphOptimizationLevel: 'all'
});
useWebGPU = false;
}
inTensorName = session.inputNames[0];
outTensorName = session.outputNames[0];
// Determine architecture type & dimensions
const isYOLO26 = modelName.startsWith('yolo26');
activeArchitecture = isYOLO26 ? 'yolo26' : 'yolo11';
// Adapt to model input dimensions if known
try {
const inputMeta = session._inputsMeta || {};
const dims = inputMeta[inTensorName]?.dimensions;
if (dims && dims[2] && typeof dims[2] === 'number') {
updateOffscreenSize(dims[2]);
}
} catch (e) {}
// Warmup probe
try {
const dummy = new Tensor('float32', new Float32Array(1 * 3 * inputSize * inputSize), [1, 3, inputSize, inputSize]);
await session.run({ [inTensorName]: dummy });
} catch (probeErr) {
console.warn('Probe notice:', probeErr);
}
badgeDot.className = 'badge-dot';
badgeText.textContent = isWebGPU ? 'WebGPU Active' : 'WASM CPU';
loaderBar.style.width = '100%';
loaderPercent.textContent = '100%';
loaderStatus.textContent = 'Neural Engine Ready';
setTimeout(() => { loaderBox.style.display = 'none'; }, 900);
showToast(`Loaded ${modelName.toUpperCase()} on ${isWebGPU ? 'WebGPU' : 'CPU'}`);
if (currentMode === 'media' && currentMediaType === 'image' && fileImage.src) {
detectAndRender(fileImage);
}
} catch (err) {
console.error('Model load error:', err);
badgeDot.className = 'badge-dot loading';
badgeText.textContent = 'Load Failed';
loaderStatus.textContent = 'Model not available';
showToast(`Notice: Run notebook export to generate ${modelName}.onnx`);
}
}
// ── Preprocessing (Letterbox with Aspect-Ratio Padding) ───────────────────────
function preprocess(source) {
const sw = source.videoWidth || source.naturalWidth || source.width || inputSize;
const sh = source.videoHeight || source.naturalHeight || source.height || inputSize;
const scale = Math.min(inputSize / sw, inputSize / sh);
const nw = Math.round(sw * scale);
const nh = Math.round(sh * scale);
const dx = Math.floor((inputSize - nw) / 2);
const dy = Math.floor((inputSize - nh) / 2);
offscreenCtx.fillStyle = 'rgb(114, 114, 114)';
offscreenCtx.fillRect(0, 0, inputSize, inputSize);
offscreenCtx.drawImage(source, dx, dy, nw, nh);
const imgData = offscreenCtx.getImageData(0, 0, inputSize, inputSize);
const data = imgData.data;
const channelSize = inputSize * inputSize;
const floatData = new Float32Array(3 * channelSize);
for (let i = 0; i < channelSize; i++) {
floatData[i] = data[4 * i] / 255.0; // R
floatData[channelSize + i] = data[4 * i + 1] / 255.0; // G
floatData[2 * channelSize + i] = data[4 * i + 2] / 255.0; // B
}
return {
tensor: new Tensor('float32', floatData, [1, 3, inputSize, inputSize]),
scale,
dx,
dy
};
}
// ── Greedy NMS for YOLO11 Anchor-Free Head ────────────────────────────────────
function runNms(boxes, scores, iouThreshold = 0.45) {
const indices = scores.map((_, i) => i).sort((a, b) => scores[b] - scores[a]);
const selected = [];
const suppressed = new Uint8Array(indices.length);
for (let i = 0; i < indices.length; i++) {
if (suppressed[i]) continue;
selected.push(indices[i]);
const [x1A, y1A, x2A, y2A] = boxes[indices[i]];
const areaA = (x2A - x1A) * (y2A - y1A);
for (let j = i + 1; j < indices.length; j++) {
if (suppressed[j]) continue;
const [x1B, y1B, x2B, y2B] = boxes[indices[j]];
const interX = Math.max(0, Math.min(x2A, x2B) - Math.max(x1A, x1B));
const interY = Math.max(0, Math.min(y2A, y2B) - Math.max(y1A, y1B));
const interArea = interX * interY;
const unionArea = areaA + ((x2B - x1B) * (y2B - y1B)) - interArea;
if (interArea / (unionArea + 1e-6) > iouThreshold) {
suppressed[j] = 1;
}
}
}
return selected;
}
// ── Apple Monotone Drawing Renderer ──────────────────────────────────────────
function renderDetections(detections) {
const count = detections.length;
metricCount.textContent = count;
ctx.lineWidth = 1.5;
ctx.font = '500 11px -apple-system, BlinkMacSystemFont, "SF Mono", monospace';
for (const det of detections) {
const { x1, y1, x2, y2, score, cls } = det;
const w = x2 - x1;
const h = y2 - y1;
// Sleek monotone bounding box
ctx.strokeStyle = 'rgba(0, 0, 0, 0.75)';
ctx.strokeRect(x1, y1, w, h);
// Apple corner accent marks
const corner = Math.min(8, w / 4, h / 4);
ctx.lineWidth = 2.5;
ctx.strokeStyle = '#000000';
ctx.beginPath();
ctx.moveTo(x1, y1 + corner); ctx.lineTo(x1, y1); ctx.lineTo(x1 + corner, y1);
ctx.moveTo(x2 - corner, y1); ctx.lineTo(x2, y1); ctx.lineTo(x2, y1 + corner);
ctx.moveTo(x1, y2 - corner); ctx.lineTo(x1, y2); ctx.lineTo(x1 + corner, y2);
ctx.moveTo(x2 - corner, y2); ctx.lineTo(x2, y2); ctx.lineTo(x2, y2 - corner);
ctx.stroke();
ctx.lineWidth = 1.5;
// Label tag
const name = COCO[cls] || `ID ${cls}`;
const tagText = `${name} ${(score * 100).toFixed(0)}%`;
const tagWidth = ctx.measureText(tagText).width + 8;
const tagHeight = 16;
const tagY = Math.max(0, y1 - tagHeight - 2);
ctx.fillStyle = 'rgba(0, 0, 0, 0.9)';
ctx.fillRect(x1, tagY, tagWidth, tagHeight);
ctx.fillStyle = '#ffffff';
ctx.fillText(tagText, x1 + 4, tagY + 12);
}
}
// ── Inference Dispatcher ─────────────────────────────────────────────────────
async function detectAndRender(source) {
if (!session) return;
const tStart = performance.now();
const { tensor, scale, dx, dy } = preprocess(source);
// Synchronize canvas dimensions with source element
const sw = source.videoWidth || source.naturalWidth || source.width || inputSize;
const sh = source.videoHeight || source.naturalHeight || source.height || inputSize;
if (displayCanvas.width !== sw || displayCanvas.height !== sh) {
displayCanvas.width = sw;
displayCanvas.height = sh;
}
const output = await session.run({ [inTensorName]: tensor });
const latency = performance.now() - tStart;
metricLatency.textContent = latency.toFixed(0);
// Compute FPS
frameCount++;
const now = performance.now();
if (now - lastTime >= 1000) {
metricFps.textContent = ((frameCount * 1000) / (now - lastTime)).toFixed(0);
frameCount = 0;
lastTime = now;
}
// Draw source frame
ctx.drawImage(source, 0, 0, sw, sh);
const confThreshold = parseFloat(confSlider.value);
const onlyPerson = personFilter.checked;
const raw = output[outTensorName];
const data = raw.data;
const dims = raw.dims;
const detections = [];
if (activeArchitecture === 'yolo26' && dims[2] === 6) {
// End-to-End NMS-Free format [1, 300, 6]
for (let i = 0; i < dims[1]; i++) {
const idx = i * 6;
const score = data[idx + 4];
if (score < confThreshold) continue;
const cls = Math.round(data[idx + 5]);
if (onlyPerson && cls !== 0) continue;
detections.push({
x1: (data[idx] - dx) / scale,
y1: (data[idx + 1] - dy) / scale,
x2: (data[idx + 2] - dx) / scale,
y2: (data[idx + 3] - dy) / scale,
score,
cls
});
}
} else {
// YOLO11 Anchor-Free format [1, 84, N]
const numCandidates = dims[2];
const candidateBoxes = [];
const candidateScores = [];
const candidateClasses = [];
for (let i = 0; i < numCandidates; i++) {
let maxScore = -1;
let maxCls = -1;
// Classes start at offset 4
for (let c = 0; c < 80; c++) {
const score = data[(4 + c) * numCandidates + i];
if (score > maxScore) {
maxScore = score;
maxCls = c;
}
}
if (maxScore >= confThreshold) {
if (onlyPerson && maxCls !== 0) continue;
const xc = data[0 * numCandidates + i];
const yc = data[1 * numCandidates + i];
const w = data[2 * numCandidates + i];
const h = data[3 * numCandidates + i];
candidateBoxes.push([xc - w / 2, yc - h / 2, xc + w / 2, yc + h / 2]);
candidateScores.push(maxScore);
candidateClasses.push(maxCls);
}
}
const nmsWinners = runNms(candidateBoxes, candidateScores, 0.45);
for (const k of nmsWinners) {
const [bx1, by1, bx2, by2] = candidateBoxes[k];
detections.push({
x1: (bx1 - dx) / scale,
y1: (by1 - dy) / scale,
x2: (bx2 - dx) / scale,
y2: (by2 - dy) / scale,
score: candidateScores[k],
cls: candidateClasses[k]
});
}
}
renderDetections(detections);
}
// ── Loops ────────────────────────────────────────────────────────────────────
async function webcamLoop() {
if (!isWebcamActive) return;
if (liveVideo.readyState >= 2) {
await detectAndRender(liveVideo);
}
animFrameId = requestAnimationFrame(webcamLoop);
}
async function videoLoop() {
if (!isVideoActive || currentMediaType !== 'video') return;
if (fileVideo.readyState >= 2 && !fileVideo.paused) {
await detectAndRender(fileVideo);
}
requestAnimationFrame(videoLoop);
}
// ── Camera Controls ──────────────────────────────────────────────────────────
startCamBtn.onclick = async () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({
video: { width: { ideal: 1280 }, height: { ideal: 720 } }
});
liveVideo.srcObject = stream;
await liveVideo.play();
isWebcamActive = true;
camIdleOverlay.style.display = 'none';
stopCamBtn.disabled = false;
startCamBtn.disabled = true;
webcamLoop();
showToast('FaceTime Camera Active');
} catch (e) {
alert('Camera permission denied or camera unavailable.');
}
};
stopCamBtn.onclick = () => {
isWebcamActive = false;
if (liveVideo.srcObject) {
liveVideo.srcObject.getTracks().forEach(t => t.stop());
liveVideo.srcObject = null;
}
if (animFrameId) cancelAnimationFrame(animFrameId);
camIdleOverlay.style.display = 'flex';
stopCamBtn.disabled = true;
startCamBtn.disabled = false;
metricLatency.textContent = '--';
metricFps.textContent = '--';
};
// ── Media File Handling ──────────────────────────────────────────────────────
function handleUploadedFile(file) {
if (!file) return;
const isVideo = file.type.startsWith('video');
const fileUrl = URL.createObjectURL(file);
mediaIdleOverlay.style.display = 'none';
mediaControls.style.display = isVideo ? 'flex' : 'none';
if (isVideo) {
currentMediaType = 'video';
fileImage.style.display = 'none';
fileVideo.src = fileUrl;
fileVideo.onloadedmetadata = () => {
displayCanvas.width = fileVideo.videoWidth;
displayCanvas.height = fileVideo.videoHeight;
fileVideo.play();
isVideoActive = true;
videoPlayBtn.textContent = 'Pause Video';
videoLoop();
};
} else {
currentMediaType = 'image';
isVideoActive = false;
fileVideo.pause();
fileVideo.src = '';
fileImage.src = fileUrl;
fileImage.onload = async () => {
displayCanvas.width = fileImage.naturalWidth;
displayCanvas.height = fileImage.naturalHeight;
await detectAndRender(fileImage);
};
}
}
mediaFileInput.onchange = (e) => handleUploadedFile(e.target.files[0]);
mediaFileInput2.onchange = (e) => handleUploadedFile(e.target.files[0]);
videoPlayBtn.onclick = () => {
if (fileVideo.paused) {
fileVideo.play();
isVideoActive = true;
videoPlayBtn.textContent = 'Pause Video';
videoLoop();
} else {
fileVideo.pause();
isVideoActive = false;
videoPlayBtn.textContent = 'Play Video';
}
};
// ── Tab Navigation ───────────────────────────────────────────────────────────
function switchTab(mode) {
currentMode = mode;
if (mode === 'webcam') {
tabWebcam.classList.add('active');
tabMedia.classList.remove('active');
webcamControls.style.display = 'flex';
mediaControls.style.display = 'none';
mediaIdleOverlay.style.display = 'none';
if (!isWebcamActive) camIdleOverlay.style.display = 'flex';
if (isVideoActive) {
fileVideo.pause();
isVideoActive = false;
}
} else {
tabMedia.classList.add('active');
tabWebcam.classList.remove('active');
webcamControls.style.display = 'none';
camIdleOverlay.style.setProperty('display', 'none', 'important');
if (!fileImage.src && !fileVideo.src) {
mediaIdleOverlay.style.setProperty('display', 'flex', 'important');
} else if (currentMediaType === 'video') {
mediaControls.style.display = 'flex';
}
}
}
tabWebcam.addEventListener('click', () => switchTab('webcam'));
tabMedia.addEventListener('click', () => switchTab('media'));
// ── Control Listeners ────────────────────────────────────────────────────────
modelPreset.addEventListener('change', () => loadSelectedModel());
devicePreset.addEventListener('change', () => loadSelectedModel());
confSlider.addEventListener('input', (e) => {
confVal.textContent = parseFloat(e.target.value).toFixed(2);
if (currentMode === 'media' && currentMediaType === 'image' && fileImage.src) {
detectAndRender(fileImage);
}
});
personFilter.addEventListener('change', () => {
if (currentMode === 'media' && currentMediaType === 'image' && fileImage.src) {
detectAndRender(fileImage);
}
});
// Initialize default model on start
loadSelectedModel();