dhvazquez's picture
Upload 6 files
12eb9ba verified
(function () {
'use strict';
/**
* Camera management module for video stream handling
*/
class CameraManager {
constructor() {
this.currentStream = null;
this.availableCameras = [];
this.selectedCameraId = null;
this.videoElement = null;
this.preferredConstraints = {
width: {
ideal: 480,
min: 240
},
height: {
ideal: 640,
min: 320
}
};
this.fallbackConstraints = {
width: {
ideal: 640,
min: 320
},
height: {
ideal: 480,
min: 240
}
};
}
/**
* Initialize camera manager and enumerate devices
* @returns {Promise<void>}
*/
async initialize() {
try {
// Request permission first
await this.requestCameraPermission();
// Enumerate cameras
await this.enumerateCameras();
console.log(`Found ${this.availableCameras.length} camera(s)`);
} catch (error) {
console.error('Failed to initialize camera manager:', error);
throw new Error(`Camera initialization failed: ${error.message}`);
}
}
/**
* Request camera permission
* @returns {Promise<void>}
*/
async requestCameraPermission() {
try {
// Request basic camera access to get permissions
const tempStream = await navigator.mediaDevices.getUserMedia({
video: true
});
// Stop the temporary stream
tempStream.getTracks().forEach(track => track.stop());
console.log('Camera permission granted');
} catch (error) {
throw new Error('Camera permission denied or not available');
}
}
/**
* Enumerate available cameras
* @returns {Promise<void>}
*/
async enumerateCameras() {
try {
const devices = await navigator.mediaDevices.enumerateDevices();
this.availableCameras = devices.filter(device => device.kind === 'videoinput').map((device, index) => ({
id: device.deviceId,
label: device.label || `Camera ${index + 1}`,
groupId: device.groupId
}));
if (this.availableCameras.length === 0) {
throw new Error('No cameras found');
}
// Select first camera by default
this.selectedCameraId = this.availableCameras[0].id;
} catch (error) {
throw new Error(`Failed to enumerate cameras: ${error.message}`);
}
}
/**
* Get list of available cameras
* @returns {Array}
*/
getAvailableCameras() {
return [...this.availableCameras];
}
/**
* Start video stream with selected camera
* @param {HTMLVideoElement} videoElement
* @param {string} cameraId
* @returns {Promise<Object>} - Stream info
*/
async startCamera(videoElement) {
let cameraId = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : null;
try {
// Stop current stream if exists
if (this.currentStream) {
this.stopCamera();
}
// Use provided camera or current selection
const targetCameraId = cameraId || this.selectedCameraId;
this.selectedCameraId = targetCameraId;
// Store video element reference
this.videoElement = videoElement;
// Try preferred resolution first (480x640)
let stream = null;
let usedConstraints = null;
try {
const constraints = {
video: {
deviceId: {
exact: targetCameraId
},
...this.preferredConstraints
}
};
console.log('Trying preferred resolution (480x640)...');
stream = await navigator.mediaDevices.getUserMedia(constraints);
usedConstraints = this.preferredConstraints;
} catch (error) {
console.log('Preferred resolution failed, trying fallback (640x480)...');
const constraints = {
video: {
deviceId: {
exact: targetCameraId
},
...this.fallbackConstraints
}
};
stream = await navigator.mediaDevices.getUserMedia(constraints);
usedConstraints = this.fallbackConstraints;
}
// Set up video element
videoElement.srcObject = stream;
this.currentStream = stream;
// Wait for video metadata to load
await new Promise((resolve, reject) => {
videoElement.addEventListener('loadedmetadata', resolve, {
once: true
});
videoElement.addEventListener('error', reject, {
once: true
});
// Timeout after 10 seconds
setTimeout(() => reject(new Error('Video load timeout')), 10000);
});
// Get actual video dimensions
const videoWidth = videoElement.videoWidth;
const videoHeight = videoElement.videoHeight;
console.log(`Video stream started: ${videoWidth}x${videoHeight}`);
// Determine if camera should be rotated for display
//const shouldRotateDisplay = ImageUtils.shouldRotateCamera(videoWidth, videoHeight);
const shouldRotateDisplay = false;
const isPortrait = videoHeight > videoWidth;
// Get video wrapper for aspect ratio handling
const videoWrapper = videoElement.closest('.video-wrapper');
// Apply rotation CSS if needed
if (shouldRotateDisplay) ; else {
videoElement.classList.remove('rotate-90ccw');
}
// Apply portrait aspect ratio if video is vertical
if (videoWrapper) {
if (isPortrait) {
videoWrapper.classList.add('portrait');
console.log('Applied portrait aspect ratio (3:4)');
} else {
videoWrapper.classList.remove('portrait');
}
}
return {
width: videoWidth,
height: videoHeight,
rotated: shouldRotateDisplay,
constraints: usedConstraints,
cameraId: targetCameraId,
cameraLabel: this.getCameraLabel(targetCameraId)
};
} catch (error) {
console.error('Failed to start camera:', error);
throw new Error(`Camera start failed: ${error.message}`);
}
}
/**
* Stop current video stream
*/
stopCamera() {
if (this.currentStream) {
this.currentStream.getTracks().forEach(track => {
track.stop();
console.log(`Stopped ${track.kind} track`);
});
this.currentStream = null;
}
if (this.videoElement) {
this.videoElement.srcObject = null;
this.videoElement.classList.remove('rotate-90ccw');
// Remove portrait class from wrapper
const videoWrapper = this.videoElement.closest('.video-wrapper');
if (videoWrapper) {
videoWrapper.classList.remove('portrait');
}
}
console.log('Camera stopped');
}
/**
* Switch to a different camera
* @param {string} cameraId
* @returns {Promise<Object>}
*/
async switchCamera(cameraId) {
if (!this.videoElement) {
throw new Error('No video element available');
}
return await this.startCamera(this.videoElement, cameraId);
}
/**
* Get label for camera ID
* @param {string} cameraId
* @returns {string}
*/
getCameraLabel(cameraId) {
const camera = this.availableCameras.find(cam => cam.id === cameraId);
return camera ? camera.label : 'Unknown Camera';
}
/**
* Get current camera information
* @returns {Object|null}
*/
getCurrentCameraInfo() {
if (!this.selectedCameraId || !this.videoElement) {
return null;
}
return {
id: this.selectedCameraId,
label: this.getCameraLabel(this.selectedCameraId),
width: this.videoElement.videoWidth,
height: this.videoElement.videoHeight,
isRotated: this.videoElement.classList.contains('rotate-90ccw')
};
}
/**
* Check if camera is currently active
* @returns {boolean}
*/
isActive() {
return this.currentStream !== null && this.currentStream.getTracks().some(track => track.readyState === 'live');
}
/**
* Get video element reference
* @returns {HTMLVideoElement|null}
*/
getVideoElement() {
return this.videoElement;
}
/**
* Set up camera selection dropdown
* @param {HTMLSelectElement} selectElement
*/
populateCameraSelect(selectElement) {
// Clear existing options
selectElement.innerHTML = '';
if (this.availableCameras.length === 0) {
const option = document.createElement('option');
option.value = '';
option.textContent = 'No cameras available';
option.disabled = true;
selectElement.appendChild(option);
return;
}
// Add camera options
this.availableCameras.forEach(camera => {
const option = document.createElement('option');
option.value = camera.id;
option.textContent = camera.label;
selectElement.appendChild(option);
});
// Select current camera
if (this.selectedCameraId) {
selectElement.value = this.selectedCameraId;
}
}
/**
* Handle device change events (camera plugged/unplugged)
* @param {Function} callback
*/
onDeviceChange(callback) {
navigator.mediaDevices.addEventListener('devicechange', async () => {
console.log('Camera devices changed');
try {
await this.enumerateCameras();
callback(this.availableCameras);
} catch (error) {
console.error('Error handling device change:', error);
callback([]);
}
});
}
/**
* Dispose of resources
*/
dispose() {
this.stopCamera();
this.availableCameras = [];
this.selectedCameraId = null;
this.videoElement = null;
console.log('Camera manager disposed');
}
}
/**
* Image utilities for preprocessing and transformations
*/
class ImageUtils {
/**
* Create a canvas element for image processing
* @param {number} width
* @param {number} height
* @returns {HTMLCanvasElement}
*/
static createCanvas(width, height) {
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
return canvas;
}
/**
* Rotate image 90 degrees counterclockwise
* @param {ImageData} imageData
* @returns {ImageData}
*/
static rotateImage90CCW(imageData) {
const {
width,
height,
data
} = imageData;
const rotatedCanvas = this.createCanvas(height, width);
const ctx = rotatedCanvas.getContext('2d');
// Create temporary canvas with original image
const tempCanvas = this.createCanvas(width, height);
const tempCtx = tempCanvas.getContext('2d');
tempCtx.putImageData(imageData, 0, 0);
// Apply rotation transformation
ctx.translate(height, 0);
ctx.rotate(Math.PI / 2);
ctx.drawImage(tempCanvas, 0, 0);
return ctx.getImageData(0, 0, height, width);
}
/**
* Resize image to target dimensions
* @param {ImageData} imageData
* @param {number} targetWidth
* @param {number} targetHeight
* @returns {ImageData}
*/
static resizeImage(imageData, targetWidth, targetHeight) {
const {
width,
height
} = imageData;
// Create canvas with original image
const sourceCanvas = this.createCanvas(width, height);
const sourceCtx = sourceCanvas.getContext('2d');
sourceCtx.putImageData(imageData, 0, 0);
// Create target canvas
const targetCanvas = this.createCanvas(targetWidth, targetHeight);
const targetCtx = targetCanvas.getContext('2d');
// Resize using canvas scaling
targetCtx.drawImage(sourceCanvas, 0, 0, width, height, 0, 0, targetWidth, targetHeight);
return targetCtx.getImageData(0, 0, targetWidth, targetHeight);
}
/**
* Extract image data from video element
* @param {HTMLVideoElement} video
* @returns {ImageData}
*/
static getVideoFrame(video) {
const canvas = this.createCanvas(video.videoWidth, video.videoHeight);
const ctx = canvas.getContext('2d');
ctx.drawImage(video, 0, 0);
return ctx.getImageData(0, 0, video.videoWidth, video.videoHeight);
}
/**
* Normalize image data for model inference
* Uses ImageNet normalization values
* @param {ImageData} imageData
* @returns {Float32Array}
*/
static normalizeImageData(imageData) {
const {
width,
height,
data
} = imageData;
const normalizedData = new Float32Array(3 * width * height);
// ImageNet normalization constants
const mean = [0.485, 0.456, 0.406];
const std = [0.229, 0.224, 0.225];
let pixelIndex = 0;
for (let i = 0; i < data.length; i += 4) {
// Extract RGB values (skip alpha channel)
const r = data[i] / 255.0;
const g = data[i + 1] / 255.0;
const b = data[i + 2] / 255.0;
// Apply normalization: (pixel - mean) / std
normalizedData[pixelIndex] = (r - mean[0]) / std[0]; // R channel
normalizedData[pixelIndex + width * height] = (g - mean[1]) / std[1]; // G channel
normalizedData[pixelIndex + 2 * width * height] = (b - mean[2]) / std[2]; // B channel
pixelIndex++;
}
return normalizedData;
}
/**
* Preprocess image for model inference
* @param {HTMLVideoElement} video
* @param {number} targetWidth
* @param {number} targetHeight
* @param {boolean} shouldRotate - Whether to rotate 90 degrees CCW
* @returns {Float32Array}
*/
static preprocessVideoFrame(video, targetWidth, targetHeight) {
try {
// Extract frame from video
let imageData = this.getVideoFrame(video);
//console.log("In", imageData.width, imageData.height);
imageData = this.cropCenter3to4(imageData);
// Rotate if needed (for 640x480 -> 480x640)
//if (shouldRotate) {
// imageData = this.rotateImage90CCW(imageData);
//}
// Resize to target dimensions
if (imageData.width !== targetWidth || imageData.height !== targetHeight) {
imageData = this.resizeImage(imageData, targetWidth, targetHeight);
}
// Normalize for model input
return this.normalizeImageData(imageData);
} catch (error) {
console.error('Error preprocessing video frame:', error);
throw error;
}
}
/**
* Create tensor from preprocessed data
* @param {Float32Array} normalizedData
* @param {number} height
* @param {number} width
* @returns {Object} - Tensor-like object for ONNX
*/
static createInputTensor(normalizedData, height, width) {
return {
data: normalizedData,
dims: [1, 3, height, width],
type: 'float32'
};
}
/**
* Process model output to create segmentation mask
* @param {Float32Array} modelOutput
* @param {number} height
* @param {number} width
* @returns {Uint8Array} - Binary mask
*/
static processModelOutput(modelOutput, height, width) {
const mask = new Uint8Array(height * width);
// Apply softmax and get argmax for each pixel
for (let i = 0; i < height * width; i++) {
const backgroundLogit = modelOutput[i];
const cardLogit = modelOutput[i + height * width];
// Simple argmax (card class = 1, background = 0)
mask[i] = cardLogit > backgroundLogit ? 255 : 0;
}
return mask;
}
/**
* Create overlay canvas with segmentation mask
* @param {Uint8Array} mask
* @param {number} width
* @param {number} height
* @param {HTMLCanvasElement} targetCanvas
* @param {boolean} shouldRotate - Whether to rotate mask back
*/
static drawSegmentationOverlay(mask, width, height, targetCanvas) {
let shouldRotate = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : false;
const ctx = targetCanvas.getContext('2d');
// Create mask canvas
const maskCanvas = this.createCanvas(width, height);
const maskCtx = maskCanvas.getContext('2d');
const maskImageData = maskCtx.createImageData(width, height);
// Fill mask with cyan color where mask is active
for (let i = 0; i < mask.length; i++) {
const pixelIndex = i * 4;
if (mask[i] > 0) {
maskImageData.data[pixelIndex] = 0; // R
maskImageData.data[pixelIndex + 1] = 255; // G (cyan)
maskImageData.data[pixelIndex + 2] = 255; // B (cyan)
maskImageData.data[pixelIndex + 3] = 128; // A (50% transparent)
} else {
maskImageData.data[pixelIndex + 3] = 0; // Fully transparent
}
}
maskCtx.putImageData(maskImageData, 0, 0);
// Clear target canvas
ctx.clearRect(0, 0, targetCanvas.width, targetCanvas.height);
shouldRotate = false;
// Draw mask with rotation if needed
if (shouldRotate) {
ctx.save();
ctx.translate(targetCanvas.width, 0);
ctx.rotate(Math.PI / 2);
ctx.drawImage(maskCanvas, 0, 0, height, width, 0, 0, targetCanvas.height, targetCanvas.width);
ctx.restore();
} else {
ctx.drawImage(maskCanvas, 0, 0, width, height, 0, 0, targetCanvas.width, targetCanvas.height);
}
}
/**
* Check if camera orientation suggests it should be rotated
* @param {number} width
* @param {number} height
* @returns {boolean}
*/
static shouldRotateCamera(width, height) {
return width > height; // Horizontal camera should be rotated to vertical
}
/**
* Check if input image needs rotation for model (640x480 -> 480x640)
* @param {number} width
* @param {number} height
* @returns {boolean}
*/
static shouldRotateForModel(width, height) {
return width === 640 && height === 480;
}
/**
* Crop the center of an image to 3:4 aspect ratio (width:height)
* Returns the largest possible crop that maintains the 3:4 proportion
* @param {ImageData} imageData
* @returns {ImageData}
*/
static cropCenter3to4(imageData) {
const {
width,
height
} = imageData;
// Calculate the largest possible 3:4 crop
let cropWidth, cropHeight;
// Determine crop dimensions based on original aspect ratio
if (width / height > 3 / 4) {
// Image is wider than 3:4, crop width
cropHeight = height;
cropWidth = Math.floor(height * 3 / 4);
} else {
// Image is taller than 3:4, crop height
cropWidth = width;
cropHeight = Math.floor(width * 4 / 3);
}
// Calculate crop position (center the crop)
const startX = Math.floor((width - cropWidth) / 2);
const startY = Math.floor((height - cropHeight) / 2);
// Create source canvas
const sourceCanvas = this.createCanvas(width, height);
const sourceCtx = sourceCanvas.getContext('2d');
sourceCtx.putImageData(imageData, 0, 0);
// Create target canvas for cropped image
const targetCanvas = this.createCanvas(cropWidth, cropHeight);
const targetCtx = targetCanvas.getContext('2d');
// Draw the cropped portion
targetCtx.drawImage(sourceCanvas, startX, startY, cropWidth, cropHeight,
// Source rectangle
0, 0, cropWidth, cropHeight // Destination rectangle
);
return targetCtx.getImageData(0, 0, cropWidth, cropHeight);
}
}
/**
* Model inference module using ONNX Runtime Web
*/
class ModelInference {
constructor() {
this.session = null;
this.isModelLoaded = false;
this.isInferring = false;
this.modelPath = 'models/card_segmentation_fp16.onnx';
// Model specifications
this.inputHeight = 320;
this.inputWidth = 240;
this.numClasses = 2;
// GPU acceleration info
this.accelerationInfo = {
activeProvider: 'unknown',
availableProviders: [],
gpuInfo: null,
supportsWebGL: false,
supportsWebGPU: false
};
// Performance tracking
this.inferenceStats = {
totalInferences: 0,
totalTime: 0,
averageTime: 0,
lastInferenceTime: 0
};
}
/**
* Initialize ONNX Runtime and load the model with GPU acceleration
* @returns {Promise<boolean>}
*/
async initialize() {
try {
console.log('Initializing ONNX Runtime with GPU acceleration...');
// Configure ONNX Runtime
ort.env.wasm.wasmPaths = 'https://cdn.jsdelivr.net/npm/onnxruntime-web@1.22.0/dist/';
ort.env.wasm.numThreads = 1; // Single thread for stability
// Detect GPU capabilities
await this.detectGPUCapabilities();
// Determine optimal execution providers
const executionProviders = this.getOptimalExecutionProviders();
console.log('Attempting to load model with providers:', executionProviders);
console.log(`Loading model from: ${this.modelPath}`);
// Try to load model with optimal providers
this.session = await this.loadModelWithFallback(executionProviders);
console.log(`Model loaded successfully with provider: ${this.accelerationInfo.activeProvider}`);
this.isModelLoaded = true;
// Log model and acceleration information
this.logModelInfo();
this.logAccelerationInfo();
return true;
} catch (error) {
console.error('Failed to initialize model:', error);
this.isModelLoaded = false;
throw new Error(`Model initialization failed: ${error.message}`);
}
}
/**
* Detect GPU capabilities and available acceleration
*/
async detectGPUCapabilities() {
console.log('Detecting GPU capabilities...');
// Check WebGL support
this.accelerationInfo.supportsWebGL = this.checkWebGLSupport();
// Check WebGPU support
this.accelerationInfo.supportsWebGPU = await this.checkWebGPUSupport();
// Get GPU info if available
if (this.accelerationInfo.supportsWebGL) {
this.accelerationInfo.gpuInfo = this.getWebGLInfo();
}
console.log('GPU Capabilities:', {
WebGL: this.accelerationInfo.supportsWebGL,
WebGPU: this.accelerationInfo.supportsWebGPU,
GPU: this.accelerationInfo.gpuInfo
});
}
/**
* Check WebGL support
* @returns {boolean}
*/
checkWebGLSupport() {
try {
const canvas = document.createElement('canvas');
const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
return !!gl;
} catch (error) {
console.warn('WebGL check failed:', error);
return false;
}
}
/**
* Check WebGPU support
* @returns {Promise<boolean>}
*/
async checkWebGPUSupport() {
try {
if (!navigator.gpu) {
return false;
}
const adapter = await navigator.gpu.requestAdapter();
return !!adapter;
} catch (error) {
console.warn('WebGPU check failed:', error);
return false;
}
}
/**
* Get WebGL renderer information
* @returns {Object|null}
*/
getWebGLInfo() {
try {
const canvas = document.createElement('canvas');
const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
if (!gl) return null;
const debugInfo = gl.getExtension('WEBGL_debug_renderer_info');
return {
vendor: debugInfo ? gl.getParameter(debugInfo.UNMASKED_VENDOR_WEBGL) : gl.getParameter(gl.VENDOR),
renderer: debugInfo ? gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL) : gl.getParameter(gl.RENDERER),
version: gl.getParameter(gl.VERSION),
shadingLanguageVersion: gl.getParameter(gl.SHADING_LANGUAGE_VERSION)
};
} catch (error) {
console.warn('Failed to get WebGL info:', error);
return null;
}
}
/**
* Determine optimal execution providers based on capabilities
* @returns {Array<string>}
*/
getOptimalExecutionProviders() {
const providers = [];
// Prioritize WebGPU first for best performance
if (this.accelerationInfo.supportsWebGPU) {
providers.push('webgpu');
this.accelerationInfo.availableProviders.push('webgpu');
}
// Fall back to WebGL if WebGPU is not available or fails
if (this.accelerationInfo.supportsWebGL) {
providers.push('webgl');
this.accelerationInfo.availableProviders.push('webgl');
}
// Always add WASM as final fallback
providers.push('wasm');
this.accelerationInfo.availableProviders.push('wasm');
return providers;
}
/**
* Load model with fallback mechanism
* @param {Array<string>} executionProviders
* @returns {Promise<InferenceSession>}
*/
async loadModelWithFallback(executionProviders) {
const baseOptions = {
enableMemPattern: false,
enableCpuMemArena: false,
graphOptimizationLevel: 'all'
};
// Try each provider in order
for (const provider of executionProviders) {
try {
console.log(`Attempting to load model with ${provider} provider...`);
const options = {
...baseOptions,
executionProviders: [provider]
};
const session = await ort.InferenceSession.create(this.modelPath, options);
this.accelerationInfo.activeProvider = provider;
console.log(`✓ Successfully loaded with ${provider} provider`);
return session;
} catch (error) {
const errorMsg = error.message;
// Log specific information for operator compatibility issues
if (errorMsg.includes('cannot resolve operator')) {
var _errorMsg$match;
const operatorName = ((_errorMsg$match = errorMsg.match(/operator '(\w+)'/)) === null || _errorMsg$match === void 0 ? void 0 : _errorMsg$match[1]) || 'unknown';
console.warn(`✗ ${provider.toUpperCase()} provider doesn't support operator '${operatorName}' - falling back to next provider`);
} else {
console.warn(`✗ Failed to load with ${provider} provider:`, errorMsg);
}
// Store failed provider info for reporting
this.accelerationInfo[`${provider}Error`] = errorMsg;
continue;
}
}
throw new Error('Failed to load model with any execution provider');
}
/**
* Log acceleration information
*/
logAccelerationInfo() {
console.log('=== Acceleration Information ===');
console.log(`Active Provider: ${this.accelerationInfo.activeProvider}`);
console.log(`Available Providers: ${this.accelerationInfo.availableProviders.join(', ')}`);
console.log(`WebGL Support: ${this.accelerationInfo.supportsWebGL}`);
console.log(`WebGPU Support: ${this.accelerationInfo.supportsWebGPU}`);
if (this.accelerationInfo.gpuInfo) {
console.log('GPU Information:');
console.log(` Vendor: ${this.accelerationInfo.gpuInfo.vendor}`);
console.log(` Renderer: ${this.accelerationInfo.gpuInfo.renderer}`);
console.log(` Version: ${this.accelerationInfo.gpuInfo.version}`);
}
console.log('===============================');
}
/**
* Get acceleration information
* @returns {Object}
*/
getAccelerationInfo() {
return {
...this.accelerationInfo
};
}
/**
* Log model information for debugging
*/
logModelInfo() {
if (!this.session) return;
console.log('=== Model Information ===');
console.log('Input tensors:');
this.session.inputNames.forEach((name, index) => {
// Try to access input metadata safely
try {
if (this.session.inputs && this.session.inputs[index]) {
const input = this.session.inputs[index];
console.log(` ${name}: ${input.dims} (${input.type})`);
} else {
console.log(` ${name}: metadata not available`);
}
} catch (error) {
console.log(` ${name}: metadata not available`);
}
});
console.log('Output tensors:');
this.session.outputNames.forEach((name, index) => {
// Try to access output metadata safely
try {
if (this.session.outputs && this.session.outputs[index]) {
const output = this.session.outputs[index];
console.log(` ${name}: ${output.dims} (${output.type})`);
} else {
console.log(` ${name}: metadata not available`);
}
} catch (error) {
console.log(` ${name}: metadata not available`);
}
});
console.log('========================');
}
/**
* Run inference on preprocessed image data
* @param {Float32Array} inputData
* @returns {Promise<Float32Array>}
*/
async runInference(inputData) {
if (!this.isModelLoaded || !this.session) {
throw new Error('Model not loaded. Call initialize() first.');
}
if (this.isInferring) {
console.warn('Inference already in progress, skipping...');
return null;
}
try {
this.isInferring = true;
const startTime = performance.now();
// Create input tensor
const inputTensor = new ort.Tensor('float32', inputData, [1, 3, this.inputHeight, this.inputWidth]);
// Get input name from model
const inputName = this.session.inputNames[0];
// Run inference
const results = await this.session.run({
[inputName]: inputTensor
});
// Get output tensor
const outputName = this.session.outputNames[0];
const outputTensor = results[outputName];
// Calculate inference time
const endTime = performance.now();
const inferenceTime = endTime - startTime;
// Update statistics
this.updateInferenceStats(inferenceTime);
console.log(`Inference completed in ${inferenceTime.toFixed(2)}ms`);
return outputTensor.data;
} catch (error) {
console.error('Inference failed:', error);
throw new Error(`Inference failed: ${error.message}`);
} finally {
this.isInferring = false;
}
}
/**
* Process video frame and return segmentation mask
* @param {HTMLVideoElement} video
* @returns {Promise<Object>} - {mask: Uint8Array, stats: Object}
*/
async processVideoFrame(video) {
try {
// Check if model is ready
if (!this.isModelLoaded) {
throw new Error('Model not initialized');
}
// Get video dimensions
const videoWidth = video.videoWidth;
const videoHeight = video.videoHeight;
if (videoWidth === 0 || videoHeight === 0) {
throw new Error('Video not ready');
}
// Determine if rotation is needed for model input
const shouldRotateForModel = ImageUtils.shouldRotateForModel(videoWidth, videoHeight);
// Preprocess video frame
const preprocessedData = ImageUtils.preprocessVideoFrame(video, this.inputWidth, this.inputHeight, shouldRotateForModel);
// Run inference
const modelOutput = await this.runInference(preprocessedData);
if (!modelOutput) {
return null; // Inference skipped
}
// Process model output to create mask
const mask = ImageUtils.processModelOutput(modelOutput, this.inputHeight, this.inputWidth);
return {
mask: mask,
shouldRotateBack: shouldRotateForModel,
stats: this.getInferenceStats()
};
} catch (error) {
console.error('Error processing video frame:', error);
throw error;
}
}
/**
* Update inference performance statistics
* @param {number} inferenceTime
*/
updateInferenceStats(inferenceTime) {
this.inferenceStats.totalInferences++;
this.inferenceStats.totalTime += inferenceTime;
this.inferenceStats.averageTime = this.inferenceStats.totalTime / this.inferenceStats.totalInferences;
this.inferenceStats.lastInferenceTime = inferenceTime;
}
/**
* Get current inference statistics
* @returns {Object}
*/
getInferenceStats() {
return {
...this.inferenceStats,
fps: this.inferenceStats.lastInferenceTime > 0 ? 1000 / this.inferenceStats.lastInferenceTime : 0
};
}
/**
* Reset inference statistics
*/
resetStats() {
this.inferenceStats = {
totalInferences: 0,
totalTime: 0,
averageTime: 0,
lastInferenceTime: 0
};
}
/**
* Check if model is ready for inference
* @returns {boolean}
*/
isReady() {
return this.isModelLoaded && !this.isInferring;
}
/**
* Get model specifications
* @returns {Object}
*/
getModelSpecs() {
return {
inputHeight: this.inputHeight,
inputWidth: this.inputWidth,
numClasses: this.numClasses,
isLoaded: this.isModelLoaded
};
}
/**
* Dispose of the model and free resources
*/
dispose() {
if (this.session) {
this.session.release();
this.session = null;
}
this.isModelLoaded = false;
this.isInferring = false;
console.log('Model resources disposed');
}
}
/**
* Main application module
* Coordinates camera, model inference, and UI interactions
*/
class CardSegmentationApp {
constructor() {
// Core modules
this.cameraManager = new CameraManager();
this.modelInference = new ModelInference();
// DOM elements
this.elements = {};
// Application state
this.state = {
isInitialized: false,
isModelLoaded: false,
isCameraActive: false,
isInferenceRunning: false,
currentError: null
};
// Animation frame ID for inference loop
this.inferenceLoopId = null;
// Performance tracking
this.performanceStats = {
frameCount: 0,
lastFpsUpdate: 0,
fps: 0
};
}
/**
* Initialize the application
*/
async init() {
try {
console.log('Initializing Card Segmentation App...');
// Get DOM elements
this.setupDOMElements();
// Set up event listeners
this.setupEventListeners();
// Show loading indicator
this.showLoading('Initializing cameras...');
// Initialize camera manager
await this.cameraManager.initialize();
this.populateCameraDropdown();
// Initialize model
this.showLoading('Loading AI model...');
await this.modelInference.initialize();
// Update acceleration info display
this.updateAccelerationInfo();
// Hide loading, show camera selection
this.hideLoading();
this.showCameraSelection();
this.state.isInitialized = true;
this.state.isModelLoaded = true;
console.log('Application initialized successfully');
} catch (error) {
console.error('Application initialization failed:', error);
this.showError('Failed to initialize application', error.message);
}
}
/**
* Set up DOM element references
*/
setupDOMElements() {
this.elements = {
// Main containers
cameraSelection: document.getElementById('cameraSelection'),
videoContainer: document.getElementById('videoContainer'),
loadingIndicator: document.getElementById('loadingIndicator'),
errorDisplay: document.getElementById('errorDisplay'),
// Camera selection
cameraSelect: document.getElementById('cameraSelect'),
startCamera: document.getElementById('startCamera'),
// Video and controls
videoElement: document.getElementById('videoElement'),
overlayCanvas: document.getElementById('overlayCanvas'),
toggleInference: document.getElementById('toggleInference'),
switchCamera: document.getElementById('switchCamera'),
// Info displays
resolutionInfo: document.getElementById('resolutionInfo'),
fpsInfo: document.getElementById('fpsInfo'),
inferenceStatus: document.getElementById('inferenceStatus'),
accelerationInfo: document.getElementById('accelerationInfo'),
// Loading and error
loadingText: document.getElementById('loadingText'),
errorMessage: document.getElementById('errorMessage'),
retryButton: document.getElementById('retryButton')
};
}
/**
* Set up event listeners
*/
setupEventListeners() {
// Camera selection
this.elements.cameraSelect.addEventListener('change', e => {
this.elements.startCamera.disabled = !e.target.value;
});
this.elements.startCamera.addEventListener('click', () => {
this.startCamera();
});
// Video controls
this.elements.toggleInference.addEventListener('click', () => {
this.toggleInference();
});
this.elements.switchCamera.addEventListener('click', () => {
this.showCameraSelection();
});
// Error handling
this.elements.retryButton.addEventListener('click', () => {
this.hideError();
this.init();
});
// Handle video loaded
this.elements.videoElement.addEventListener('loadedmetadata', () => {
this.setupOverlayCanvas();
});
// Handle camera device changes
this.cameraManager.onDeviceChange(() => {
this.populateCameraDropdown();
});
// Handle page unload
window.addEventListener('beforeunload', () => {
this.cleanup();
});
}
/**
* Populate camera selection dropdown
*/
populateCameraDropdown() {
this.cameraManager.populateCameraSelect(this.elements.cameraSelect);
this.elements.startCamera.disabled = this.elements.cameraSelect.value === '';
}
/**
* Start camera with selected device
*/
async startCamera() {
try {
const selectedCameraId = this.elements.cameraSelect.value;
if (!selectedCameraId) {
throw new Error('No camera selected');
}
this.showLoading('Starting camera...');
// Start camera stream
const streamInfo = await this.cameraManager.startCamera(this.elements.videoElement, selectedCameraId);
// Update UI
this.updateResolutionInfo(streamInfo);
this.setupOverlayCanvas();
// Show video container
this.hideLoading();
this.hideCameraSelection();
this.showVideoContainer();
this.state.isCameraActive = true;
console.log('Camera started successfully:', streamInfo);
} catch (error) {
console.error('Failed to start camera:', error);
this.showError('Failed to start camera', error.message);
}
}
/**
* Set up overlay canvas to match video dimensions
*/
setupOverlayCanvas() {
const video = this.elements.videoElement;
const canvas = this.elements.overlayCanvas;
if (video.videoWidth && video.videoHeight) {
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
// Apply same rotation class as video if needed
if (video.classList.contains('rotate-90ccw')) {
canvas.classList.add('rotate-90ccw');
} else {
canvas.classList.remove('rotate-90ccw');
}
}
}
/**
* Toggle inference on/off
*/
toggleInference() {
if (this.state.isInferenceRunning) {
this.stopInference();
} else {
this.startInference();
}
}
/**
* Start continuous inference
*/
startInference() {
if (!this.state.isModelLoaded || !this.state.isCameraActive) {
this.showError('Cannot start inference', 'Camera or model not ready');
return;
}
this.state.isInferenceRunning = true;
this.elements.toggleInference.textContent = 'Stop Detection';
this.elements.toggleInference.className = 'btn btn-danger';
this.updateInferenceStatus('Detection: Running');
// Reset performance stats
this.performanceStats.frameCount = 0;
this.performanceStats.lastFpsUpdate = performance.now();
// Start inference loop
this.runInferenceLoop();
console.log('Inference started');
}
/**
* Stop continuous inference
*/
stopInference() {
this.state.isInferenceRunning = false;
this.elements.toggleInference.textContent = 'Start Detection';
this.elements.toggleInference.className = 'btn btn-success';
this.updateInferenceStatus('Detection: Off');
// Cancel animation frame
if (this.inferenceLoopId) {
cancelAnimationFrame(this.inferenceLoopId);
this.inferenceLoopId = null;
}
// Clear overlay
const ctx = this.elements.overlayCanvas.getContext('2d');
ctx.clearRect(0, 0, this.elements.overlayCanvas.width, this.elements.overlayCanvas.height);
console.log('Inference stopped');
}
/**
* Main inference loop
*/
async runInferenceLoop() {
if (!this.state.isInferenceRunning) {
return;
}
try {
// Process video frame
const result = await this.modelInference.processVideoFrame(this.elements.videoElement);
if (result && result.mask) {
// Draw segmentation overlay
ImageUtils.drawSegmentationOverlay(result.mask, this.modelInference.inputWidth, this.modelInference.inputHeight, this.elements.overlayCanvas, result.shouldRotateBack);
// Update performance stats
this.updatePerformanceStats(result.stats);
}
} catch (error) {
console.error('Inference loop error:', error);
// Don't stop inference for individual frame errors
}
// Schedule next frame
if (this.state.isInferenceRunning) {
this.inferenceLoopId = requestAnimationFrame(() => this.runInferenceLoop());
}
}
/**
* Update performance statistics display
*/
updatePerformanceStats(inferenceStats) {
this.performanceStats.frameCount++;
const now = performance.now();
// Update FPS every second
if (now - this.performanceStats.lastFpsUpdate >= 1000) {
this.performanceStats.fps = this.performanceStats.frameCount;
this.performanceStats.frameCount = 0;
this.performanceStats.lastFpsUpdate = now;
// Update UI
this.elements.fpsInfo.textContent = `FPS: ${this.performanceStats.fps}`;
}
}
/**
* Update resolution information display
*/
updateResolutionInfo(streamInfo) {
const resText = `Resolution: ${streamInfo.width}x${streamInfo.height}${streamInfo.rotated ? ' (rotated)' : ''}`;
this.elements.resolutionInfo.textContent = resText;
}
/**
* Update inference status display
*/
updateInferenceStatus(status) {
this.elements.inferenceStatus.textContent = status;
// Update status styling
this.elements.inferenceStatus.className = 'status-inactive';
if (status.includes('Running')) {
this.elements.inferenceStatus.className = 'status-active';
} else if (status.includes('Processing')) {
this.elements.inferenceStatus.className = 'status-processing';
}
}
/**
* Update acceleration information display
*/
updateAccelerationInfo() {
if (!this.elements.accelerationInfo) return;
const accelerationInfo = this.modelInference.getAccelerationInfo();
// Create acceleration status text
let accelerationText = `Acceleration: ${accelerationInfo.activeProvider.toUpperCase()}`;
// Add GPU info if available
if (accelerationInfo.gpuInfo && accelerationInfo.activeProvider !== 'wasm') {
const gpu = accelerationInfo.gpuInfo;
const vendor = gpu.vendor.includes('Google') ? gpu.renderer.split(' ')[0] : gpu.vendor;
accelerationText += ` (${vendor})`;
}
// Add note if using CPU fallback due to GPU limitations
if (accelerationInfo.activeProvider === 'wasm') {
if (accelerationInfo.webglError && accelerationInfo.webglError.includes('cannot resolve operator')) {
accelerationText += ' (GPU unsupported operators)';
} else if (accelerationInfo.supportsWebGL || accelerationInfo.supportsWebGPU) {
accelerationText += ' (GPU provider failed)';
} else {
accelerationText += ' (No GPU support)';
}
}
this.elements.accelerationInfo.textContent = accelerationText;
// Update styling based on acceleration type
this.elements.accelerationInfo.className = 'acceleration-info';
if (accelerationInfo.activeProvider === 'webgpu') {
this.elements.accelerationInfo.classList.add('acceleration-webgpu');
} else if (accelerationInfo.activeProvider === 'webgl') {
this.elements.accelerationInfo.classList.add('acceleration-webgl');
} else {
this.elements.accelerationInfo.classList.add('acceleration-wasm');
}
}
/**
* Show loading indicator
*/
showLoading() {
let message = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 'Loading...';
this.elements.loadingText.textContent = message;
this.elements.loadingIndicator.style.display = 'block';
this.elements.cameraSelection.style.display = 'none';
this.elements.videoContainer.style.display = 'none';
this.elements.errorDisplay.style.display = 'none';
}
/**
* Hide loading indicator
*/
hideLoading() {
this.elements.loadingIndicator.style.display = 'none';
}
/**
* Show camera selection
*/
showCameraSelection() {
this.stopInference();
this.cameraManager.stopCamera();
this.state.isCameraActive = false;
this.elements.cameraSelection.style.display = 'block';
this.elements.videoContainer.style.display = 'none';
this.elements.errorDisplay.style.display = 'none';
}
/**
* Hide camera selection
*/
hideCameraSelection() {
this.elements.cameraSelection.style.display = 'none';
}
/**
* Show video container
*/
showVideoContainer() {
this.elements.videoContainer.style.display = 'block';
this.elements.errorDisplay.style.display = 'none';
}
/**
* Show error message
*/
showError(title, message) {
this.elements.errorMessage.innerHTML = `<strong>${title}</strong><br>${message}`;
this.elements.errorDisplay.style.display = 'block';
this.elements.loadingIndicator.style.display = 'none';
this.elements.cameraSelection.style.display = 'none';
this.elements.videoContainer.style.display = 'none';
this.state.currentError = {
title,
message
};
}
/**
* Hide error display
*/
hideError() {
this.elements.errorDisplay.style.display = 'none';
this.state.currentError = null;
}
/**
* Clean up resources
*/
cleanup() {
console.log('Cleaning up application...');
this.stopInference();
this.cameraManager.dispose();
this.modelInference.dispose();
this.state = {
isInitialized: false,
isModelLoaded: false,
isCameraActive: false,
isInferenceRunning: false,
currentError: null
};
}
}
// Initialize application when DOM is loaded
document.addEventListener('DOMContentLoaded', () => {
const app = new CardSegmentationApp();
app.init();
// Make app globally available for debugging
window.cardSegmentationApp = app;
});
})();
//# sourceMappingURL=bundle.js.map