(function () { 'use strict'; /** * Camera management module for video stream handling */ class CameraManager { constructor() { this.currentStream = null; this.availableCameras = []; this.selectedCameraId = null; this.videoElement = null; this.preferredConstraints = { width: { ideal: 480, min: 240 }, height: { ideal: 640, min: 320 } }; this.fallbackConstraints = { width: { ideal: 640, min: 320 }, height: { ideal: 480, min: 240 } }; } /** * Initialize camera manager and enumerate devices * @returns {Promise} */ async initialize() { try { // Request permission first await this.requestCameraPermission(); // Enumerate cameras await this.enumerateCameras(); console.log(`Found ${this.availableCameras.length} camera(s)`); } catch (error) { console.error('Failed to initialize camera manager:', error); throw new Error(`Camera initialization failed: ${error.message}`); } } /** * Request camera permission * @returns {Promise} */ async requestCameraPermission() { try { // Request basic camera access to get permissions const tempStream = await navigator.mediaDevices.getUserMedia({ video: true }); // Stop the temporary stream tempStream.getTracks().forEach(track => track.stop()); console.log('Camera permission granted'); } catch (error) { throw new Error('Camera permission denied or not available'); } } /** * Enumerate available cameras * @returns {Promise} */ async enumerateCameras() { try { const devices = await navigator.mediaDevices.enumerateDevices(); this.availableCameras = devices.filter(device => device.kind === 'videoinput').map((device, index) => ({ id: device.deviceId, label: device.label || `Camera ${index + 1}`, groupId: device.groupId })); if (this.availableCameras.length === 0) { throw new Error('No cameras found'); } // Select first camera by default this.selectedCameraId = this.availableCameras[0].id; } catch (error) { throw new Error(`Failed to enumerate cameras: ${error.message}`); } } /** * Get list of available cameras * @returns {Array} */ getAvailableCameras() { return [...this.availableCameras]; } /** * Start video stream with selected camera * @param {HTMLVideoElement} videoElement * @param {string} cameraId * @returns {Promise} - Stream info */ async startCamera(videoElement) { let cameraId = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : null; try { // Stop current stream if exists if (this.currentStream) { this.stopCamera(); } // Use provided camera or current selection const targetCameraId = cameraId || this.selectedCameraId; this.selectedCameraId = targetCameraId; // Store video element reference this.videoElement = videoElement; // Try preferred resolution first (480x640) let stream = null; let usedConstraints = null; try { const constraints = { video: { deviceId: { exact: targetCameraId }, ...this.preferredConstraints } }; console.log('Trying preferred resolution (480x640)...'); stream = await navigator.mediaDevices.getUserMedia(constraints); usedConstraints = this.preferredConstraints; } catch (error) { console.log('Preferred resolution failed, trying fallback (640x480)...'); const constraints = { video: { deviceId: { exact: targetCameraId }, ...this.fallbackConstraints } }; stream = await navigator.mediaDevices.getUserMedia(constraints); usedConstraints = this.fallbackConstraints; } // Set up video element videoElement.srcObject = stream; this.currentStream = stream; // Wait for video metadata to load await new Promise((resolve, reject) => { videoElement.addEventListener('loadedmetadata', resolve, { once: true }); videoElement.addEventListener('error', reject, { once: true }); // Timeout after 10 seconds setTimeout(() => reject(new Error('Video load timeout')), 10000); }); // Get actual video dimensions const videoWidth = videoElement.videoWidth; const videoHeight = videoElement.videoHeight; console.log(`Video stream started: ${videoWidth}x${videoHeight}`); // Determine if camera should be rotated for display //const shouldRotateDisplay = ImageUtils.shouldRotateCamera(videoWidth, videoHeight); const shouldRotateDisplay = false; const isPortrait = videoHeight > videoWidth; // Get video wrapper for aspect ratio handling const videoWrapper = videoElement.closest('.video-wrapper'); // Apply rotation CSS if needed if (shouldRotateDisplay) ; else { videoElement.classList.remove('rotate-90ccw'); } // Apply portrait aspect ratio if video is vertical if (videoWrapper) { if (isPortrait) { videoWrapper.classList.add('portrait'); console.log('Applied portrait aspect ratio (3:4)'); } else { videoWrapper.classList.remove('portrait'); } } return { width: videoWidth, height: videoHeight, rotated: shouldRotateDisplay, constraints: usedConstraints, cameraId: targetCameraId, cameraLabel: this.getCameraLabel(targetCameraId) }; } catch (error) { console.error('Failed to start camera:', error); throw new Error(`Camera start failed: ${error.message}`); } } /** * Stop current video stream */ stopCamera() { if (this.currentStream) { this.currentStream.getTracks().forEach(track => { track.stop(); console.log(`Stopped ${track.kind} track`); }); this.currentStream = null; } if (this.videoElement) { this.videoElement.srcObject = null; this.videoElement.classList.remove('rotate-90ccw'); // Remove portrait class from wrapper const videoWrapper = this.videoElement.closest('.video-wrapper'); if (videoWrapper) { videoWrapper.classList.remove('portrait'); } } console.log('Camera stopped'); } /** * Switch to a different camera * @param {string} cameraId * @returns {Promise} */ async switchCamera(cameraId) { if (!this.videoElement) { throw new Error('No video element available'); } return await this.startCamera(this.videoElement, cameraId); } /** * Get label for camera ID * @param {string} cameraId * @returns {string} */ getCameraLabel(cameraId) { const camera = this.availableCameras.find(cam => cam.id === cameraId); return camera ? camera.label : 'Unknown Camera'; } /** * Get current camera information * @returns {Object|null} */ getCurrentCameraInfo() { if (!this.selectedCameraId || !this.videoElement) { return null; } return { id: this.selectedCameraId, label: this.getCameraLabel(this.selectedCameraId), width: this.videoElement.videoWidth, height: this.videoElement.videoHeight, isRotated: this.videoElement.classList.contains('rotate-90ccw') }; } /** * Check if camera is currently active * @returns {boolean} */ isActive() { return this.currentStream !== null && this.currentStream.getTracks().some(track => track.readyState === 'live'); } /** * Get video element reference * @returns {HTMLVideoElement|null} */ getVideoElement() { return this.videoElement; } /** * Set up camera selection dropdown * @param {HTMLSelectElement} selectElement */ populateCameraSelect(selectElement) { // Clear existing options selectElement.innerHTML = ''; if (this.availableCameras.length === 0) { const option = document.createElement('option'); option.value = ''; option.textContent = 'No cameras available'; option.disabled = true; selectElement.appendChild(option); return; } // Add camera options this.availableCameras.forEach(camera => { const option = document.createElement('option'); option.value = camera.id; option.textContent = camera.label; selectElement.appendChild(option); }); // Select current camera if (this.selectedCameraId) { selectElement.value = this.selectedCameraId; } } /** * Handle device change events (camera plugged/unplugged) * @param {Function} callback */ onDeviceChange(callback) { navigator.mediaDevices.addEventListener('devicechange', async () => { console.log('Camera devices changed'); try { await this.enumerateCameras(); callback(this.availableCameras); } catch (error) { console.error('Error handling device change:', error); callback([]); } }); } /** * Dispose of resources */ dispose() { this.stopCamera(); this.availableCameras = []; this.selectedCameraId = null; this.videoElement = null; console.log('Camera manager disposed'); } } /** * Image utilities for preprocessing and transformations */ class ImageUtils { /** * Create a canvas element for image processing * @param {number} width * @param {number} height * @returns {HTMLCanvasElement} */ static createCanvas(width, height) { const canvas = document.createElement('canvas'); canvas.width = width; canvas.height = height; return canvas; } /** * Rotate image 90 degrees counterclockwise * @param {ImageData} imageData * @returns {ImageData} */ static rotateImage90CCW(imageData) { const { width, height, data } = imageData; const rotatedCanvas = this.createCanvas(height, width); const ctx = rotatedCanvas.getContext('2d'); // Create temporary canvas with original image const tempCanvas = this.createCanvas(width, height); const tempCtx = tempCanvas.getContext('2d'); tempCtx.putImageData(imageData, 0, 0); // Apply rotation transformation ctx.translate(height, 0); ctx.rotate(Math.PI / 2); ctx.drawImage(tempCanvas, 0, 0); return ctx.getImageData(0, 0, height, width); } /** * Resize image to target dimensions * @param {ImageData} imageData * @param {number} targetWidth * @param {number} targetHeight * @returns {ImageData} */ static resizeImage(imageData, targetWidth, targetHeight) { const { width, height } = imageData; // Create canvas with original image const sourceCanvas = this.createCanvas(width, height); const sourceCtx = sourceCanvas.getContext('2d'); sourceCtx.putImageData(imageData, 0, 0); // Create target canvas const targetCanvas = this.createCanvas(targetWidth, targetHeight); const targetCtx = targetCanvas.getContext('2d'); // Resize using canvas scaling targetCtx.drawImage(sourceCanvas, 0, 0, width, height, 0, 0, targetWidth, targetHeight); return targetCtx.getImageData(0, 0, targetWidth, targetHeight); } /** * Extract image data from video element * @param {HTMLVideoElement} video * @returns {ImageData} */ static getVideoFrame(video) { const canvas = this.createCanvas(video.videoWidth, video.videoHeight); const ctx = canvas.getContext('2d'); ctx.drawImage(video, 0, 0); return ctx.getImageData(0, 0, video.videoWidth, video.videoHeight); } /** * Normalize image data for model inference * Uses ImageNet normalization values * @param {ImageData} imageData * @returns {Float32Array} */ static normalizeImageData(imageData) { const { width, height, data } = imageData; const normalizedData = new Float32Array(3 * width * height); // ImageNet normalization constants const mean = [0.485, 0.456, 0.406]; const std = [0.229, 0.224, 0.225]; let pixelIndex = 0; for (let i = 0; i < data.length; i += 4) { // Extract RGB values (skip alpha channel) const r = data[i] / 255.0; const g = data[i + 1] / 255.0; const b = data[i + 2] / 255.0; // Apply normalization: (pixel - mean) / std normalizedData[pixelIndex] = (r - mean[0]) / std[0]; // R channel normalizedData[pixelIndex + width * height] = (g - mean[1]) / std[1]; // G channel normalizedData[pixelIndex + 2 * width * height] = (b - mean[2]) / std[2]; // B channel pixelIndex++; } return normalizedData; } /** * Preprocess image for model inference * @param {HTMLVideoElement} video * @param {number} targetWidth * @param {number} targetHeight * @param {boolean} shouldRotate - Whether to rotate 90 degrees CCW * @returns {Float32Array} */ static preprocessVideoFrame(video, targetWidth, targetHeight) { try { // Extract frame from video let imageData = this.getVideoFrame(video); //console.log("In", imageData.width, imageData.height); imageData = this.cropCenter3to4(imageData); // Rotate if needed (for 640x480 -> 480x640) //if (shouldRotate) { // imageData = this.rotateImage90CCW(imageData); //} // Resize to target dimensions if (imageData.width !== targetWidth || imageData.height !== targetHeight) { imageData = this.resizeImage(imageData, targetWidth, targetHeight); } // Normalize for model input return this.normalizeImageData(imageData); } catch (error) { console.error('Error preprocessing video frame:', error); throw error; } } /** * Create tensor from preprocessed data * @param {Float32Array} normalizedData * @param {number} height * @param {number} width * @returns {Object} - Tensor-like object for ONNX */ static createInputTensor(normalizedData, height, width) { return { data: normalizedData, dims: [1, 3, height, width], type: 'float32' }; } /** * Process model output to create segmentation mask * @param {Float32Array} modelOutput * @param {number} height * @param {number} width * @returns {Uint8Array} - Binary mask */ static processModelOutput(modelOutput, height, width) { const mask = new Uint8Array(height * width); // Apply softmax and get argmax for each pixel for (let i = 0; i < height * width; i++) { const backgroundLogit = modelOutput[i]; const cardLogit = modelOutput[i + height * width]; // Simple argmax (card class = 1, background = 0) mask[i] = cardLogit > backgroundLogit ? 255 : 0; } return mask; } /** * Create overlay canvas with segmentation mask * @param {Uint8Array} mask * @param {number} width * @param {number} height * @param {HTMLCanvasElement} targetCanvas * @param {boolean} shouldRotate - Whether to rotate mask back */ static drawSegmentationOverlay(mask, width, height, targetCanvas) { let shouldRotate = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : false; const ctx = targetCanvas.getContext('2d'); // Create mask canvas const maskCanvas = this.createCanvas(width, height); const maskCtx = maskCanvas.getContext('2d'); const maskImageData = maskCtx.createImageData(width, height); // Fill mask with cyan color where mask is active for (let i = 0; i < mask.length; i++) { const pixelIndex = i * 4; if (mask[i] > 0) { maskImageData.data[pixelIndex] = 0; // R maskImageData.data[pixelIndex + 1] = 255; // G (cyan) maskImageData.data[pixelIndex + 2] = 255; // B (cyan) maskImageData.data[pixelIndex + 3] = 128; // A (50% transparent) } else { maskImageData.data[pixelIndex + 3] = 0; // Fully transparent } } maskCtx.putImageData(maskImageData, 0, 0); // Clear target canvas ctx.clearRect(0, 0, targetCanvas.width, targetCanvas.height); shouldRotate = false; // Draw mask with rotation if needed if (shouldRotate) { ctx.save(); ctx.translate(targetCanvas.width, 0); ctx.rotate(Math.PI / 2); ctx.drawImage(maskCanvas, 0, 0, height, width, 0, 0, targetCanvas.height, targetCanvas.width); ctx.restore(); } else { ctx.drawImage(maskCanvas, 0, 0, width, height, 0, 0, targetCanvas.width, targetCanvas.height); } } /** * Check if camera orientation suggests it should be rotated * @param {number} width * @param {number} height * @returns {boolean} */ static shouldRotateCamera(width, height) { return width > height; // Horizontal camera should be rotated to vertical } /** * Check if input image needs rotation for model (640x480 -> 480x640) * @param {number} width * @param {number} height * @returns {boolean} */ static shouldRotateForModel(width, height) { return width === 640 && height === 480; } /** * Crop the center of an image to 3:4 aspect ratio (width:height) * Returns the largest possible crop that maintains the 3:4 proportion * @param {ImageData} imageData * @returns {ImageData} */ static cropCenter3to4(imageData) { const { width, height } = imageData; // Calculate the largest possible 3:4 crop let cropWidth, cropHeight; // Determine crop dimensions based on original aspect ratio if (width / height > 3 / 4) { // Image is wider than 3:4, crop width cropHeight = height; cropWidth = Math.floor(height * 3 / 4); } else { // Image is taller than 3:4, crop height cropWidth = width; cropHeight = Math.floor(width * 4 / 3); } // Calculate crop position (center the crop) const startX = Math.floor((width - cropWidth) / 2); const startY = Math.floor((height - cropHeight) / 2); // Create source canvas const sourceCanvas = this.createCanvas(width, height); const sourceCtx = sourceCanvas.getContext('2d'); sourceCtx.putImageData(imageData, 0, 0); // Create target canvas for cropped image const targetCanvas = this.createCanvas(cropWidth, cropHeight); const targetCtx = targetCanvas.getContext('2d'); // Draw the cropped portion targetCtx.drawImage(sourceCanvas, startX, startY, cropWidth, cropHeight, // Source rectangle 0, 0, cropWidth, cropHeight // Destination rectangle ); return targetCtx.getImageData(0, 0, cropWidth, cropHeight); } } /** * Model inference module using ONNX Runtime Web */ class ModelInference { constructor() { this.session = null; this.isModelLoaded = false; this.isInferring = false; this.modelPath = 'models/card_segmentation_fp16.onnx'; // Model specifications this.inputHeight = 320; this.inputWidth = 240; this.numClasses = 2; // GPU acceleration info this.accelerationInfo = { activeProvider: 'unknown', availableProviders: [], gpuInfo: null, supportsWebGL: false, supportsWebGPU: false }; // Performance tracking this.inferenceStats = { totalInferences: 0, totalTime: 0, averageTime: 0, lastInferenceTime: 0 }; } /** * Initialize ONNX Runtime and load the model with GPU acceleration * @returns {Promise} */ async initialize() { try { console.log('Initializing ONNX Runtime with GPU acceleration...'); // Configure ONNX Runtime ort.env.wasm.wasmPaths = 'https://cdn.jsdelivr.net/npm/onnxruntime-web@1.22.0/dist/'; ort.env.wasm.numThreads = 1; // Single thread for stability // Detect GPU capabilities await this.detectGPUCapabilities(); // Determine optimal execution providers const executionProviders = this.getOptimalExecutionProviders(); console.log('Attempting to load model with providers:', executionProviders); console.log(`Loading model from: ${this.modelPath}`); // Try to load model with optimal providers this.session = await this.loadModelWithFallback(executionProviders); console.log(`Model loaded successfully with provider: ${this.accelerationInfo.activeProvider}`); this.isModelLoaded = true; // Log model and acceleration information this.logModelInfo(); this.logAccelerationInfo(); return true; } catch (error) { console.error('Failed to initialize model:', error); this.isModelLoaded = false; throw new Error(`Model initialization failed: ${error.message}`); } } /** * Detect GPU capabilities and available acceleration */ async detectGPUCapabilities() { console.log('Detecting GPU capabilities...'); // Check WebGL support this.accelerationInfo.supportsWebGL = this.checkWebGLSupport(); // Check WebGPU support this.accelerationInfo.supportsWebGPU = await this.checkWebGPUSupport(); // Get GPU info if available if (this.accelerationInfo.supportsWebGL) { this.accelerationInfo.gpuInfo = this.getWebGLInfo(); } console.log('GPU Capabilities:', { WebGL: this.accelerationInfo.supportsWebGL, WebGPU: this.accelerationInfo.supportsWebGPU, GPU: this.accelerationInfo.gpuInfo }); } /** * Check WebGL support * @returns {boolean} */ checkWebGLSupport() { try { const canvas = document.createElement('canvas'); const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl'); return !!gl; } catch (error) { console.warn('WebGL check failed:', error); return false; } } /** * Check WebGPU support * @returns {Promise} */ async checkWebGPUSupport() { try { if (!navigator.gpu) { return false; } const adapter = await navigator.gpu.requestAdapter(); return !!adapter; } catch (error) { console.warn('WebGPU check failed:', error); return false; } } /** * Get WebGL renderer information * @returns {Object|null} */ getWebGLInfo() { try { const canvas = document.createElement('canvas'); const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl'); if (!gl) return null; const debugInfo = gl.getExtension('WEBGL_debug_renderer_info'); return { vendor: debugInfo ? gl.getParameter(debugInfo.UNMASKED_VENDOR_WEBGL) : gl.getParameter(gl.VENDOR), renderer: debugInfo ? gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL) : gl.getParameter(gl.RENDERER), version: gl.getParameter(gl.VERSION), shadingLanguageVersion: gl.getParameter(gl.SHADING_LANGUAGE_VERSION) }; } catch (error) { console.warn('Failed to get WebGL info:', error); return null; } } /** * Determine optimal execution providers based on capabilities * @returns {Array} */ getOptimalExecutionProviders() { const providers = []; // Prioritize WebGPU first for best performance if (this.accelerationInfo.supportsWebGPU) { providers.push('webgpu'); this.accelerationInfo.availableProviders.push('webgpu'); } // Fall back to WebGL if WebGPU is not available or fails if (this.accelerationInfo.supportsWebGL) { providers.push('webgl'); this.accelerationInfo.availableProviders.push('webgl'); } // Always add WASM as final fallback providers.push('wasm'); this.accelerationInfo.availableProviders.push('wasm'); return providers; } /** * Load model with fallback mechanism * @param {Array} executionProviders * @returns {Promise} */ async loadModelWithFallback(executionProviders) { const baseOptions = { enableMemPattern: false, enableCpuMemArena: false, graphOptimizationLevel: 'all' }; // Try each provider in order for (const provider of executionProviders) { try { console.log(`Attempting to load model with ${provider} provider...`); const options = { ...baseOptions, executionProviders: [provider] }; const session = await ort.InferenceSession.create(this.modelPath, options); this.accelerationInfo.activeProvider = provider; console.log(`✓ Successfully loaded with ${provider} provider`); return session; } catch (error) { const errorMsg = error.message; // Log specific information for operator compatibility issues if (errorMsg.includes('cannot resolve operator')) { var _errorMsg$match; const operatorName = ((_errorMsg$match = errorMsg.match(/operator '(\w+)'/)) === null || _errorMsg$match === void 0 ? void 0 : _errorMsg$match[1]) || 'unknown'; console.warn(`✗ ${provider.toUpperCase()} provider doesn't support operator '${operatorName}' - falling back to next provider`); } else { console.warn(`✗ Failed to load with ${provider} provider:`, errorMsg); } // Store failed provider info for reporting this.accelerationInfo[`${provider}Error`] = errorMsg; continue; } } throw new Error('Failed to load model with any execution provider'); } /** * Log acceleration information */ logAccelerationInfo() { console.log('=== Acceleration Information ==='); console.log(`Active Provider: ${this.accelerationInfo.activeProvider}`); console.log(`Available Providers: ${this.accelerationInfo.availableProviders.join(', ')}`); console.log(`WebGL Support: ${this.accelerationInfo.supportsWebGL}`); console.log(`WebGPU Support: ${this.accelerationInfo.supportsWebGPU}`); if (this.accelerationInfo.gpuInfo) { console.log('GPU Information:'); console.log(` Vendor: ${this.accelerationInfo.gpuInfo.vendor}`); console.log(` Renderer: ${this.accelerationInfo.gpuInfo.renderer}`); console.log(` Version: ${this.accelerationInfo.gpuInfo.version}`); } console.log('==============================='); } /** * Get acceleration information * @returns {Object} */ getAccelerationInfo() { return { ...this.accelerationInfo }; } /** * Log model information for debugging */ logModelInfo() { if (!this.session) return; console.log('=== Model Information ==='); console.log('Input tensors:'); this.session.inputNames.forEach((name, index) => { // Try to access input metadata safely try { if (this.session.inputs && this.session.inputs[index]) { const input = this.session.inputs[index]; console.log(` ${name}: ${input.dims} (${input.type})`); } else { console.log(` ${name}: metadata not available`); } } catch (error) { console.log(` ${name}: metadata not available`); } }); console.log('Output tensors:'); this.session.outputNames.forEach((name, index) => { // Try to access output metadata safely try { if (this.session.outputs && this.session.outputs[index]) { const output = this.session.outputs[index]; console.log(` ${name}: ${output.dims} (${output.type})`); } else { console.log(` ${name}: metadata not available`); } } catch (error) { console.log(` ${name}: metadata not available`); } }); console.log('========================'); } /** * Run inference on preprocessed image data * @param {Float32Array} inputData * @returns {Promise} */ async runInference(inputData) { if (!this.isModelLoaded || !this.session) { throw new Error('Model not loaded. Call initialize() first.'); } if (this.isInferring) { console.warn('Inference already in progress, skipping...'); return null; } try { this.isInferring = true; const startTime = performance.now(); // Create input tensor const inputTensor = new ort.Tensor('float32', inputData, [1, 3, this.inputHeight, this.inputWidth]); // Get input name from model const inputName = this.session.inputNames[0]; // Run inference const results = await this.session.run({ [inputName]: inputTensor }); // Get output tensor const outputName = this.session.outputNames[0]; const outputTensor = results[outputName]; // Calculate inference time const endTime = performance.now(); const inferenceTime = endTime - startTime; // Update statistics this.updateInferenceStats(inferenceTime); console.log(`Inference completed in ${inferenceTime.toFixed(2)}ms`); return outputTensor.data; } catch (error) { console.error('Inference failed:', error); throw new Error(`Inference failed: ${error.message}`); } finally { this.isInferring = false; } } /** * Process video frame and return segmentation mask * @param {HTMLVideoElement} video * @returns {Promise} - {mask: Uint8Array, stats: Object} */ async processVideoFrame(video) { try { // Check if model is ready if (!this.isModelLoaded) { throw new Error('Model not initialized'); } // Get video dimensions const videoWidth = video.videoWidth; const videoHeight = video.videoHeight; if (videoWidth === 0 || videoHeight === 0) { throw new Error('Video not ready'); } // Determine if rotation is needed for model input const shouldRotateForModel = ImageUtils.shouldRotateForModel(videoWidth, videoHeight); // Preprocess video frame const preprocessedData = ImageUtils.preprocessVideoFrame(video, this.inputWidth, this.inputHeight, shouldRotateForModel); // Run inference const modelOutput = await this.runInference(preprocessedData); if (!modelOutput) { return null; // Inference skipped } // Process model output to create mask const mask = ImageUtils.processModelOutput(modelOutput, this.inputHeight, this.inputWidth); return { mask: mask, shouldRotateBack: shouldRotateForModel, stats: this.getInferenceStats() }; } catch (error) { console.error('Error processing video frame:', error); throw error; } } /** * Update inference performance statistics * @param {number} inferenceTime */ updateInferenceStats(inferenceTime) { this.inferenceStats.totalInferences++; this.inferenceStats.totalTime += inferenceTime; this.inferenceStats.averageTime = this.inferenceStats.totalTime / this.inferenceStats.totalInferences; this.inferenceStats.lastInferenceTime = inferenceTime; } /** * Get current inference statistics * @returns {Object} */ getInferenceStats() { return { ...this.inferenceStats, fps: this.inferenceStats.lastInferenceTime > 0 ? 1000 / this.inferenceStats.lastInferenceTime : 0 }; } /** * Reset inference statistics */ resetStats() { this.inferenceStats = { totalInferences: 0, totalTime: 0, averageTime: 0, lastInferenceTime: 0 }; } /** * Check if model is ready for inference * @returns {boolean} */ isReady() { return this.isModelLoaded && !this.isInferring; } /** * Get model specifications * @returns {Object} */ getModelSpecs() { return { inputHeight: this.inputHeight, inputWidth: this.inputWidth, numClasses: this.numClasses, isLoaded: this.isModelLoaded }; } /** * Dispose of the model and free resources */ dispose() { if (this.session) { this.session.release(); this.session = null; } this.isModelLoaded = false; this.isInferring = false; console.log('Model resources disposed'); } } /** * Main application module * Coordinates camera, model inference, and UI interactions */ class CardSegmentationApp { constructor() { // Core modules this.cameraManager = new CameraManager(); this.modelInference = new ModelInference(); // DOM elements this.elements = {}; // Application state this.state = { isInitialized: false, isModelLoaded: false, isCameraActive: false, isInferenceRunning: false, currentError: null }; // Animation frame ID for inference loop this.inferenceLoopId = null; // Performance tracking this.performanceStats = { frameCount: 0, lastFpsUpdate: 0, fps: 0 }; } /** * Initialize the application */ async init() { try { console.log('Initializing Card Segmentation App...'); // Get DOM elements this.setupDOMElements(); // Set up event listeners this.setupEventListeners(); // Show loading indicator this.showLoading('Initializing cameras...'); // Initialize camera manager await this.cameraManager.initialize(); this.populateCameraDropdown(); // Initialize model this.showLoading('Loading AI model...'); await this.modelInference.initialize(); // Update acceleration info display this.updateAccelerationInfo(); // Hide loading, show camera selection this.hideLoading(); this.showCameraSelection(); this.state.isInitialized = true; this.state.isModelLoaded = true; console.log('Application initialized successfully'); } catch (error) { console.error('Application initialization failed:', error); this.showError('Failed to initialize application', error.message); } } /** * Set up DOM element references */ setupDOMElements() { this.elements = { // Main containers cameraSelection: document.getElementById('cameraSelection'), videoContainer: document.getElementById('videoContainer'), loadingIndicator: document.getElementById('loadingIndicator'), errorDisplay: document.getElementById('errorDisplay'), // Camera selection cameraSelect: document.getElementById('cameraSelect'), startCamera: document.getElementById('startCamera'), // Video and controls videoElement: document.getElementById('videoElement'), overlayCanvas: document.getElementById('overlayCanvas'), toggleInference: document.getElementById('toggleInference'), switchCamera: document.getElementById('switchCamera'), // Info displays resolutionInfo: document.getElementById('resolutionInfo'), fpsInfo: document.getElementById('fpsInfo'), inferenceStatus: document.getElementById('inferenceStatus'), accelerationInfo: document.getElementById('accelerationInfo'), // Loading and error loadingText: document.getElementById('loadingText'), errorMessage: document.getElementById('errorMessage'), retryButton: document.getElementById('retryButton') }; } /** * Set up event listeners */ setupEventListeners() { // Camera selection this.elements.cameraSelect.addEventListener('change', e => { this.elements.startCamera.disabled = !e.target.value; }); this.elements.startCamera.addEventListener('click', () => { this.startCamera(); }); // Video controls this.elements.toggleInference.addEventListener('click', () => { this.toggleInference(); }); this.elements.switchCamera.addEventListener('click', () => { this.showCameraSelection(); }); // Error handling this.elements.retryButton.addEventListener('click', () => { this.hideError(); this.init(); }); // Handle video loaded this.elements.videoElement.addEventListener('loadedmetadata', () => { this.setupOverlayCanvas(); }); // Handle camera device changes this.cameraManager.onDeviceChange(() => { this.populateCameraDropdown(); }); // Handle page unload window.addEventListener('beforeunload', () => { this.cleanup(); }); } /** * Populate camera selection dropdown */ populateCameraDropdown() { this.cameraManager.populateCameraSelect(this.elements.cameraSelect); this.elements.startCamera.disabled = this.elements.cameraSelect.value === ''; } /** * Start camera with selected device */ async startCamera() { try { const selectedCameraId = this.elements.cameraSelect.value; if (!selectedCameraId) { throw new Error('No camera selected'); } this.showLoading('Starting camera...'); // Start camera stream const streamInfo = await this.cameraManager.startCamera(this.elements.videoElement, selectedCameraId); // Update UI this.updateResolutionInfo(streamInfo); this.setupOverlayCanvas(); // Show video container this.hideLoading(); this.hideCameraSelection(); this.showVideoContainer(); this.state.isCameraActive = true; console.log('Camera started successfully:', streamInfo); } catch (error) { console.error('Failed to start camera:', error); this.showError('Failed to start camera', error.message); } } /** * Set up overlay canvas to match video dimensions */ setupOverlayCanvas() { const video = this.elements.videoElement; const canvas = this.elements.overlayCanvas; if (video.videoWidth && video.videoHeight) { canvas.width = video.videoWidth; canvas.height = video.videoHeight; // Apply same rotation class as video if needed if (video.classList.contains('rotate-90ccw')) { canvas.classList.add('rotate-90ccw'); } else { canvas.classList.remove('rotate-90ccw'); } } } /** * Toggle inference on/off */ toggleInference() { if (this.state.isInferenceRunning) { this.stopInference(); } else { this.startInference(); } } /** * Start continuous inference */ startInference() { if (!this.state.isModelLoaded || !this.state.isCameraActive) { this.showError('Cannot start inference', 'Camera or model not ready'); return; } this.state.isInferenceRunning = true; this.elements.toggleInference.textContent = 'Stop Detection'; this.elements.toggleInference.className = 'btn btn-danger'; this.updateInferenceStatus('Detection: Running'); // Reset performance stats this.performanceStats.frameCount = 0; this.performanceStats.lastFpsUpdate = performance.now(); // Start inference loop this.runInferenceLoop(); console.log('Inference started'); } /** * Stop continuous inference */ stopInference() { this.state.isInferenceRunning = false; this.elements.toggleInference.textContent = 'Start Detection'; this.elements.toggleInference.className = 'btn btn-success'; this.updateInferenceStatus('Detection: Off'); // Cancel animation frame if (this.inferenceLoopId) { cancelAnimationFrame(this.inferenceLoopId); this.inferenceLoopId = null; } // Clear overlay const ctx = this.elements.overlayCanvas.getContext('2d'); ctx.clearRect(0, 0, this.elements.overlayCanvas.width, this.elements.overlayCanvas.height); console.log('Inference stopped'); } /** * Main inference loop */ async runInferenceLoop() { if (!this.state.isInferenceRunning) { return; } try { // Process video frame const result = await this.modelInference.processVideoFrame(this.elements.videoElement); if (result && result.mask) { // Draw segmentation overlay ImageUtils.drawSegmentationOverlay(result.mask, this.modelInference.inputWidth, this.modelInference.inputHeight, this.elements.overlayCanvas, result.shouldRotateBack); // Update performance stats this.updatePerformanceStats(result.stats); } } catch (error) { console.error('Inference loop error:', error); // Don't stop inference for individual frame errors } // Schedule next frame if (this.state.isInferenceRunning) { this.inferenceLoopId = requestAnimationFrame(() => this.runInferenceLoop()); } } /** * Update performance statistics display */ updatePerformanceStats(inferenceStats) { this.performanceStats.frameCount++; const now = performance.now(); // Update FPS every second if (now - this.performanceStats.lastFpsUpdate >= 1000) { this.performanceStats.fps = this.performanceStats.frameCount; this.performanceStats.frameCount = 0; this.performanceStats.lastFpsUpdate = now; // Update UI this.elements.fpsInfo.textContent = `FPS: ${this.performanceStats.fps}`; } } /** * Update resolution information display */ updateResolutionInfo(streamInfo) { const resText = `Resolution: ${streamInfo.width}x${streamInfo.height}${streamInfo.rotated ? ' (rotated)' : ''}`; this.elements.resolutionInfo.textContent = resText; } /** * Update inference status display */ updateInferenceStatus(status) { this.elements.inferenceStatus.textContent = status; // Update status styling this.elements.inferenceStatus.className = 'status-inactive'; if (status.includes('Running')) { this.elements.inferenceStatus.className = 'status-active'; } else if (status.includes('Processing')) { this.elements.inferenceStatus.className = 'status-processing'; } } /** * Update acceleration information display */ updateAccelerationInfo() { if (!this.elements.accelerationInfo) return; const accelerationInfo = this.modelInference.getAccelerationInfo(); // Create acceleration status text let accelerationText = `Acceleration: ${accelerationInfo.activeProvider.toUpperCase()}`; // Add GPU info if available if (accelerationInfo.gpuInfo && accelerationInfo.activeProvider !== 'wasm') { const gpu = accelerationInfo.gpuInfo; const vendor = gpu.vendor.includes('Google') ? gpu.renderer.split(' ')[0] : gpu.vendor; accelerationText += ` (${vendor})`; } // Add note if using CPU fallback due to GPU limitations if (accelerationInfo.activeProvider === 'wasm') { if (accelerationInfo.webglError && accelerationInfo.webglError.includes('cannot resolve operator')) { accelerationText += ' (GPU unsupported operators)'; } else if (accelerationInfo.supportsWebGL || accelerationInfo.supportsWebGPU) { accelerationText += ' (GPU provider failed)'; } else { accelerationText += ' (No GPU support)'; } } this.elements.accelerationInfo.textContent = accelerationText; // Update styling based on acceleration type this.elements.accelerationInfo.className = 'acceleration-info'; if (accelerationInfo.activeProvider === 'webgpu') { this.elements.accelerationInfo.classList.add('acceleration-webgpu'); } else if (accelerationInfo.activeProvider === 'webgl') { this.elements.accelerationInfo.classList.add('acceleration-webgl'); } else { this.elements.accelerationInfo.classList.add('acceleration-wasm'); } } /** * Show loading indicator */ showLoading() { let message = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 'Loading...'; this.elements.loadingText.textContent = message; this.elements.loadingIndicator.style.display = 'block'; this.elements.cameraSelection.style.display = 'none'; this.elements.videoContainer.style.display = 'none'; this.elements.errorDisplay.style.display = 'none'; } /** * Hide loading indicator */ hideLoading() { this.elements.loadingIndicator.style.display = 'none'; } /** * Show camera selection */ showCameraSelection() { this.stopInference(); this.cameraManager.stopCamera(); this.state.isCameraActive = false; this.elements.cameraSelection.style.display = 'block'; this.elements.videoContainer.style.display = 'none'; this.elements.errorDisplay.style.display = 'none'; } /** * Hide camera selection */ hideCameraSelection() { this.elements.cameraSelection.style.display = 'none'; } /** * Show video container */ showVideoContainer() { this.elements.videoContainer.style.display = 'block'; this.elements.errorDisplay.style.display = 'none'; } /** * Show error message */ showError(title, message) { this.elements.errorMessage.innerHTML = `${title}
${message}`; this.elements.errorDisplay.style.display = 'block'; this.elements.loadingIndicator.style.display = 'none'; this.elements.cameraSelection.style.display = 'none'; this.elements.videoContainer.style.display = 'none'; this.state.currentError = { title, message }; } /** * Hide error display */ hideError() { this.elements.errorDisplay.style.display = 'none'; this.state.currentError = null; } /** * Clean up resources */ cleanup() { console.log('Cleaning up application...'); this.stopInference(); this.cameraManager.dispose(); this.modelInference.dispose(); this.state = { isInitialized: false, isModelLoaded: false, isCameraActive: false, isInferenceRunning: false, currentError: null }; } } // Initialize application when DOM is loaded document.addEventListener('DOMContentLoaded', () => { const app = new CardSegmentationApp(); app.init(); // Make app globally available for debugging window.cardSegmentationApp = app; }); })(); //# sourceMappingURL=bundle.js.map