Spaces:
Running
Running
| ; | |
| /* | |
| ============================================================ | |
| LocateAnything Live V2 | |
| Smooth Camera + Async CPU Inference | |
| ============================================================ | |
| */ | |
| const state = { | |
| stream: null, | |
| running: false, | |
| connected: false, | |
| processing: false, | |
| currentJob: null, | |
| frameId: 0, | |
| query: "person", | |
| mode: "hybrid", | |
| pollTimer: null, | |
| captureTimer: null, | |
| reconnectTimer: null, | |
| captureInterval: 1200, | |
| detections: [], | |
| inferenceStarted: 0, | |
| fpsFrames: 0, | |
| fpsStarted: performance.now(), | |
| inferenceCount: 0, | |
| inferenceFpsStarted: performance.now(), | |
| }; | |
| /* ========================================================= | |
| ELEMENTS | |
| ========================================================= */ | |
| const el = { | |
| camera: | |
| document.getElementById("camera"), | |
| overlay: | |
| document.getElementById("overlay"), | |
| cameraPlaceholder: | |
| document.getElementById( | |
| "cameraPlaceholder" | |
| ), | |
| startButton: | |
| document.getElementById( | |
| "startButton" | |
| ), | |
| stopButton: | |
| document.getElementById( | |
| "stopButton" | |
| ), | |
| snapshotButton: | |
| document.getElementById( | |
| "snapshotButton" | |
| ), | |
| mobileStart: | |
| document.getElementById( | |
| "mobileStart" | |
| ), | |
| mobileSnapshot: | |
| document.getElementById( | |
| "mobileSnapshot" | |
| ), | |
| imageUpload: | |
| document.getElementById( | |
| "imageUpload" | |
| ), | |
| query: | |
| document.getElementById( | |
| "query" | |
| ), | |
| clearQuery: | |
| document.getElementById( | |
| "clearQuery" | |
| ), | |
| chips: | |
| document.querySelectorAll( | |
| ".chip" | |
| ), | |
| modes: | |
| document.querySelectorAll( | |
| ".mode-button" | |
| ), | |
| connectionDot: | |
| document.getElementById( | |
| "connectionDot" | |
| ), | |
| connectionText: | |
| document.getElementById( | |
| "connectionText" | |
| ), | |
| liveBadge: | |
| document.getElementById( | |
| "liveBadge" | |
| ), | |
| fpsBadge: | |
| document.getElementById( | |
| "fpsBadge" | |
| ), | |
| reticle: | |
| document.getElementById( | |
| "reticle" | |
| ), | |
| objectCount: | |
| document.getElementById( | |
| "objectCount" | |
| ), | |
| latency: | |
| document.getElementById( | |
| "latency" | |
| ), | |
| inferenceFps: | |
| document.getElementById( | |
| "inferenceFps" | |
| ), | |
| statusTitle: | |
| document.getElementById( | |
| "statusTitle" | |
| ), | |
| statusMessage: | |
| document.getElementById( | |
| "statusMessage" | |
| ), | |
| resultsPanel: | |
| document.getElementById( | |
| "resultsPanel" | |
| ), | |
| resultsCount: | |
| document.getElementById( | |
| "resultsCount" | |
| ), | |
| resultsList: | |
| document.getElementById( | |
| "resultsList" | |
| ), | |
| permissionModal: | |
| document.getElementById( | |
| "permissionModal" | |
| ), | |
| permissionClose: | |
| document.getElementById( | |
| "permissionClose" | |
| ), | |
| }; | |
| /* ========================================================= | |
| SAFE ELEMENT HELPERS | |
| ========================================================= */ | |
| function exists(element) { | |
| return !!element; | |
| } | |
| function text(element, value) { | |
| if (element) { | |
| element.textContent = value; | |
| } | |
| } | |
| function toggle( | |
| element, | |
| className, | |
| enabled | |
| ) { | |
| if (!element) { | |
| return; | |
| } | |
| element.classList.toggle( | |
| className, | |
| enabled | |
| ); | |
| } | |
| /* ========================================================= | |
| STATUS | |
| ========================================================= */ | |
| function setConnection( | |
| status, | |
| message | |
| ) { | |
| state.connected = | |
| status === "online"; | |
| if (el.connectionDot) { | |
| el.connectionDot.className = | |
| `connection-dot ${status}`; | |
| } | |
| text( | |
| el.connectionText, | |
| message | |
| ); | |
| } | |
| function setStatus( | |
| title, | |
| message | |
| ) { | |
| text( | |
| el.statusTitle, | |
| title | |
| ); | |
| text( | |
| el.statusMessage, | |
| message | |
| ); | |
| } | |
| function setLive( | |
| enabled | |
| ) { | |
| toggle( | |
| el.liveBadge, | |
| "hidden", | |
| !enabled | |
| ); | |
| toggle( | |
| el.fpsBadge, | |
| "hidden", | |
| !enabled | |
| ); | |
| toggle( | |
| el.reticle, | |
| "hidden", | |
| !enabled | |
| ); | |
| } | |
| function updateButtons() { | |
| if (el.startButton) { | |
| el.startButton.disabled = | |
| state.running; | |
| } | |
| if (el.stopButton) { | |
| el.stopButton.disabled = | |
| !state.running; | |
| } | |
| if (el.snapshotButton) { | |
| el.snapshotButton.disabled = | |
| !state.running; | |
| } | |
| if (el.mobileSnapshot) { | |
| el.mobileSnapshot.disabled = | |
| !state.running; | |
| } | |
| if (el.mobileStart) { | |
| el.mobileStart.textContent = | |
| state.running | |
| ? "Stop" | |
| : "Start"; | |
| } | |
| } | |
| /* ========================================================= | |
| WEBSOCKET | |
| ========================================================= */ | |
| function websocketURL() { | |
| const protocol = | |
| location.protocol === "https:" | |
| ? "wss:" | |
| : "ws:"; | |
| return ( | |
| `${protocol}//${location.host}/ws` | |
| ); | |
| } | |
| function connectSocket() { | |
| if ( | |
| state.socket && | |
| ( | |
| state.socket.readyState === | |
| WebSocket.OPEN | |
| || | |
| state.socket.readyState === | |
| WebSocket.CONNECTING | |
| ) | |
| ) { | |
| return; | |
| } | |
| setConnection( | |
| "offline", | |
| "Connecting" | |
| ); | |
| try { | |
| state.socket = | |
| new WebSocket( | |
| websocketURL() | |
| ); | |
| } catch (error) { | |
| console.error(error); | |
| scheduleReconnect(); | |
| return; | |
| } | |
| state.socket.onopen = () => { | |
| console.log( | |
| "LocateAnything WebSocket connected." | |
| ); | |
| setConnection( | |
| "online", | |
| "Connected" | |
| ); | |
| if (state.running) { | |
| setStatus( | |
| "Camera active", | |
| `Searching for “${state.query}”.` | |
| ); | |
| } else { | |
| setStatus( | |
| "Ready to locate", | |
| "Start the camera to begin." | |
| ); | |
| } | |
| }; | |
| state.socket.onmessage = ( | |
| event | |
| ) => { | |
| handleServerMessage( | |
| event.data | |
| ); | |
| }; | |
| state.socket.onerror = ( | |
| error | |
| ) => { | |
| console.error( | |
| "WebSocket error:", | |
| error | |
| ); | |
| setConnection( | |
| "error", | |
| "Connection error" | |
| ); | |
| }; | |
| state.socket.onclose = () => { | |
| state.connected = | |
| false; | |
| setConnection( | |
| "offline", | |
| "Disconnected" | |
| ); | |
| if (state.running) { | |
| setStatus( | |
| "Reconnecting", | |
| "Reconnecting to the server…" | |
| ); | |
| scheduleReconnect(); | |
| } | |
| }; | |
| } | |
| function scheduleReconnect() { | |
| if (state.reconnectTimer) { | |
| return; | |
| } | |
| state.reconnectTimer = | |
| setTimeout( | |
| () => { | |
| state.reconnectTimer = | |
| null; | |
| connectSocket(); | |
| }, | |
| 1500 | |
| ); | |
| } | |
| /* ========================================================= | |
| SERVER MESSAGES | |
| ========================================================= */ | |
| function handleServerMessage( | |
| raw | |
| ) { | |
| let message; | |
| try { | |
| message = | |
| JSON.parse(raw); | |
| } catch (error) { | |
| console.error( | |
| "Invalid server JSON:", | |
| raw | |
| ); | |
| return; | |
| } | |
| console.log( | |
| "LocateAnything:", | |
| message | |
| ); | |
| switch ( | |
| message.type | |
| ) { | |
| case "connected": | |
| state.connected = | |
| true; | |
| setConnection( | |
| "online", | |
| "Connected" | |
| ); | |
| break; | |
| case "job": | |
| handleJobCreated( | |
| message | |
| ); | |
| break; | |
| case "busy": | |
| handleBusy( | |
| message | |
| ); | |
| break; | |
| case "processing": | |
| setStatus( | |
| "Analyzing", | |
| `Locating “${state.query}”…` | |
| ); | |
| break; | |
| case "result": | |
| /* | |
| * Compatibility with the older backend. | |
| */ | |
| handleResult( | |
| message | |
| ); | |
| break; | |
| case "error": | |
| handleError( | |
| message | |
| ); | |
| break; | |
| case "stopped": | |
| state.processing = | |
| false; | |
| state.currentJob = | |
| null; | |
| break; | |
| case "pong": | |
| break; | |
| default: | |
| console.log( | |
| "Unknown server message:", | |
| message | |
| ); | |
| } | |
| } | |
| /* ========================================================= | |
| JOB CREATED | |
| ========================================================= */ | |
| function handleJobCreated( | |
| message | |
| ) { | |
| const jobId = | |
| message.job_id; | |
| if (!jobId) { | |
| state.processing = | |
| false; | |
| setStatus( | |
| "Job error", | |
| "The server did not return a job ID." | |
| ); | |
| return; | |
| } | |
| state.currentJob = | |
| jobId; | |
| state.processing = | |
| true; | |
| state.inferenceStarted = | |
| performance.now(); | |
| setStatus( | |
| "Analyzing", | |
| `Locating “${state.query}”…` | |
| ); | |
| /* | |
| * Start polling. | |
| * | |
| * The camera itself continues running. | |
| */ | |
| pollJob( | |
| jobId | |
| ); | |
| } | |
| /* ========================================================= | |
| POLL JOB | |
| ========================================================= */ | |
| async function pollJob( | |
| jobId | |
| ) { | |
| if ( | |
| !state.processing || | |
| state.currentJob !== jobId | |
| ) { | |
| return; | |
| } | |
| try { | |
| const response = | |
| await fetch( | |
| `/job/${encodeURIComponent( | |
| jobId | |
| )}`, | |
| { | |
| method: "GET", | |
| cache: "no-store", | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw new Error( | |
| `HTTP ${response.status}` | |
| ); | |
| } | |
| const job = | |
| await response.json(); | |
| if ( | |
| job.status === | |
| "queued" | |
| ) { | |
| updateJobTimer(); | |
| scheduleJobPoll( | |
| jobId, | |
| 1500 | |
| ); | |
| return; | |
| } | |
| if ( | |
| job.status === | |
| "running" | |
| ) { | |
| updateJobTimer(); | |
| scheduleJobPoll( | |
| jobId, | |
| 2000 | |
| ); | |
| return; | |
| } | |
| if ( | |
| job.status === | |
| "completed" | |
| ) { | |
| state.processing = | |
| false; | |
| state.currentJob = | |
| null; | |
| handleResult( | |
| job | |
| ); | |
| /* | |
| * Give the camera a moment before | |
| * submitting the next frame. | |
| */ | |
| scheduleCapture( | |
| 800 | |
| ); | |
| return; | |
| } | |
| if ( | |
| job.status === | |
| "error" | |
| ) { | |
| state.processing = | |
| false; | |
| state.currentJob = | |
| null; | |
| handleError( | |
| job | |
| ); | |
| scheduleCapture( | |
| 1500 | |
| ); | |
| return; | |
| } | |
| scheduleJobPoll( | |
| jobId, | |
| 1500 | |
| ); | |
| } catch (error) { | |
| console.error( | |
| "Job polling error:", | |
| error | |
| ); | |
| /* | |
| * Don't immediately kill the job. | |
| * | |
| * The CPU inference can still be running. | |
| */ | |
| scheduleJobPoll( | |
| jobId, | |
| 3000 | |
| ); | |
| } | |
| } | |
| function scheduleJobPoll( | |
| jobId, | |
| delay | |
| ) { | |
| if (state.pollTimer) { | |
| clearTimeout( | |
| state.pollTimer | |
| ); | |
| } | |
| state.pollTimer = | |
| setTimeout( | |
| () => { | |
| state.pollTimer = | |
| null; | |
| pollJob( | |
| jobId | |
| ); | |
| }, | |
| delay | |
| ); | |
| } | |
| /* ========================================================= | |
| INFERENCE TIMER | |
| ========================================================= */ | |
| function updateJobTimer() { | |
| if ( | |
| !state.inferenceStarted | |
| ) { | |
| return; | |
| } | |
| const elapsed = | |
| performance.now() | |
| - | |
| state.inferenceStarted; | |
| const seconds = | |
| elapsed / 1000; | |
| setStatus( | |
| "Analyzing", | |
| `Locating “${state.query}”… ${seconds.toFixed(0)}s` | |
| ); | |
| } | |
| /* ========================================================= | |
| RESULT | |
| ========================================================= */ | |
| function handleResult( | |
| message | |
| ) { | |
| const detections = | |
| Array.isArray( | |
| message.detections | |
| ) | |
| ? message.detections | |
| : []; | |
| state.detections = | |
| detections; | |
| state.inferenceCount++; | |
| text( | |
| el.objectCount, | |
| String( | |
| detections.length | |
| ) | |
| ); | |
| if ( | |
| Number.isFinite( | |
| Number( | |
| message.latency | |
| ) | |
| ) | |
| ) { | |
| text( | |
| el.latency, | |
| `${( | |
| Number( | |
| message.latency | |
| ) / 1000 | |
| ).toFixed(1)}s` | |
| ); | |
| } | |
| updateInferenceFPS(); | |
| updateResults( | |
| detections | |
| ); | |
| drawOverlay(); | |
| if ( | |
| detections.length | |
| ) { | |
| setStatus( | |
| `${detections.length} ${ | |
| detections.length === 1 | |
| ? "object" | |
| : "objects" | |
| } located`, | |
| `Latest result for “${state.query}”.` | |
| ); | |
| } else { | |
| setStatus( | |
| "Nothing located", | |
| `No matching “${state.query}” was found in the latest frame.` | |
| ); | |
| } | |
| } | |
| /* ========================================================= | |
| ERROR | |
| ========================================================= */ | |
| function handleError( | |
| message | |
| ) { | |
| console.error( | |
| "LocateAnything error:", | |
| message | |
| ); | |
| setStatus( | |
| "Inference error", | |
| message.detail || | |
| message.message || | |
| "Inference failed." | |
| ); | |
| } | |
| /* ========================================================= | |
| BUSY | |
| ========================================================= */ | |
| function handleBusy( | |
| message | |
| ) { | |
| state.processing = | |
| true; | |
| setStatus( | |
| "Analyzing", | |
| message.message || | |
| "Previous inference is still running." | |
| ); | |
| } | |
| /* ========================================================= | |
| CAMERA | |
| ========================================================= */ | |
| async function startCamera() { | |
| if (state.running) { | |
| stopCamera(); | |
| return; | |
| } | |
| try { | |
| if ( | |
| !navigator.mediaDevices || | |
| !navigator.mediaDevices | |
| .getUserMedia | |
| ) { | |
| throw new Error( | |
| "Camera API is unavailable." | |
| ); | |
| } | |
| const stream = | |
| await navigator | |
| .mediaDevices | |
| .getUserMedia( | |
| { | |
| video: { | |
| facingMode: { | |
| ideal: | |
| "environment" | |
| }, | |
| width: { | |
| ideal: 1280 | |
| }, | |
| height: { | |
| ideal: 720 | |
| }, | |
| frameRate: { | |
| ideal: 30, | |
| max: 30 | |
| } | |
| }, | |
| audio: false | |
| } | |
| ); | |
| state.stream = | |
| stream; | |
| el.camera.srcObject = | |
| stream; | |
| await el.camera.play(); | |
| state.running = | |
| true; | |
| state.processing = | |
| false; | |
| if ( | |
| el.cameraPlaceholder | |
| ) { | |
| el.cameraPlaceholder | |
| .classList | |
| .add( | |
| "hidden" | |
| ); | |
| } | |
| setLive( | |
| true | |
| ); | |
| updateButtons(); | |
| setStatus( | |
| "Camera active", | |
| `Searching for “${state.query}”.` | |
| ); | |
| connectSocket(); | |
| startCameraFPS(); | |
| resizeCanvas(); | |
| /* | |
| * Submit the first frame shortly | |
| * after the camera starts. | |
| */ | |
| scheduleCapture( | |
| 1000 | |
| ); | |
| } catch (error) { | |
| console.error( | |
| "Camera error:", | |
| error | |
| ); | |
| if ( | |
| el.permissionModal | |
| ) { | |
| el.permissionModal | |
| .classList | |
| .remove( | |
| "hidden" | |
| ); | |
| } | |
| setStatus( | |
| "Camera unavailable", | |
| error.message || | |
| "Please allow camera access." | |
| ); | |
| } | |
| } | |
| /* ========================================================= | |
| STOP | |
| ========================================================= */ | |
| function stopCamera() { | |
| state.running = | |
| false; | |
| state.processing = | |
| false; | |
| state.currentJob = | |
| null; | |
| if ( | |
| state.captureTimer | |
| ) { | |
| clearTimeout( | |
| state.captureTimer | |
| ); | |
| state.captureTimer = | |
| null; | |
| } | |
| if ( | |
| state.pollTimer | |
| ) { | |
| clearTimeout( | |
| state.pollTimer | |
| ); | |
| state.pollTimer = | |
| null; | |
| } | |
| if ( | |
| state.stream | |
| ) { | |
| state.stream | |
| .getTracks() | |
| .forEach( | |
| track => | |
| track.stop() | |
| ); | |
| state.stream = | |
| null; | |
| } | |
| if ( | |
| el.camera | |
| ) { | |
| el.camera.srcObject = | |
| null; | |
| } | |
| state.detections = | |
| []; | |
| clearOverlay(); | |
| setLive( | |
| false | |
| ); | |
| updateButtons(); | |
| if ( | |
| el.cameraPlaceholder | |
| ) { | |
| el.cameraPlaceholder | |
| .classList | |
| .remove( | |
| "hidden" | |
| ); | |
| } | |
| setStatus( | |
| "Camera stopped", | |
| "Start the camera to continue." | |
| ); | |
| if ( | |
| state.socket && | |
| state.socket.readyState === | |
| WebSocket.OPEN | |
| ) { | |
| try { | |
| state.socket.send( | |
| JSON.stringify( | |
| { | |
| type: | |
| "stop" | |
| } | |
| ) | |
| ); | |
| } catch (error) { | |
| console.warn( | |
| error | |
| ); | |
| } | |
| } | |
| } | |
| /* ========================================================= | |
| CAMERA CAPTURE | |
| ========================================================= */ | |
| function scheduleCapture( | |
| delay | |
| ) { | |
| if (!state.running) { | |
| return; | |
| } | |
| if ( | |
| state.captureTimer | |
| ) { | |
| clearTimeout( | |
| state.captureTimer | |
| ); | |
| } | |
| state.captureTimer = | |
| setTimeout( | |
| () => { | |
| state.captureTimer = | |
| null; | |
| captureFrame(); | |
| }, | |
| delay | |
| ); | |
| } | |
| function captureFrame() { | |
| if (!state.running) { | |
| return; | |
| } | |
| /* | |
| * Never submit another frame while | |
| * the previous CPU inference is running. | |
| */ | |
| if (state.processing) { | |
| return; | |
| } | |
| if ( | |
| !state.socket || | |
| state.socket.readyState !== | |
| WebSocket.OPEN | |
| ) { | |
| connectSocket(); | |
| scheduleCapture( | |
| 2000 | |
| ); | |
| return; | |
| } | |
| const video = | |
| el.camera; | |
| if ( | |
| !video.videoWidth || | |
| !video.videoHeight | |
| ) { | |
| scheduleCapture( | |
| 1000 | |
| ); | |
| return; | |
| } | |
| const canvas = | |
| document.createElement( | |
| "canvas" | |
| ); | |
| const maximum = | |
| 768; | |
| const ratio = | |
| Math.min( | |
| 1, | |
| maximum / | |
| Math.max( | |
| video.videoWidth, | |
| video.videoHeight | |
| ) | |
| ); | |
| canvas.width = | |
| Math.round( | |
| video.videoWidth * | |
| ratio | |
| ); | |
| canvas.height = | |
| Math.round( | |
| video.videoHeight * | |
| ratio | |
| ); | |
| const context = | |
| canvas.getContext( | |
| "2d", | |
| { | |
| alpha: false | |
| } | |
| ); | |
| context.drawImage( | |
| video, | |
| 0, | |
| 0, | |
| canvas.width, | |
| canvas.height | |
| ); | |
| const image = | |
| canvas.toDataURL( | |
| "image/jpeg", | |
| 0.68 | |
| ); | |
| state.frameId++; | |
| try { | |
| state.socket.send( | |
| JSON.stringify( | |
| { | |
| type: | |
| "frame", | |
| frame_id: | |
| state.frameId, | |
| image, | |
| query: | |
| state.query, | |
| mode: | |
| state.mode | |
| } | |
| ) | |
| ); | |
| /* | |
| * The server will send "job". | |
| * | |
| * Do NOT schedule another capture | |
| * here. The next frame is scheduled | |
| * after the current job completes. | |
| */ | |
| } catch (error) { | |
| console.error( | |
| "Frame send failed:", | |
| error | |
| ); | |
| scheduleCapture( | |
| 2000 | |
| ); | |
| } | |
| } | |
| /* ========================================================= | |
| SNAPSHOT | |
| ========================================================= */ | |
| function snapshot() { | |
| if ( | |
| !state.running || | |
| !el.camera.videoWidth | |
| ) { | |
| return; | |
| } | |
| const canvas = | |
| document.createElement( | |
| "canvas" | |
| ); | |
| canvas.width = | |
| el.camera.videoWidth; | |
| canvas.height = | |
| el.camera.videoHeight; | |
| const context = | |
| canvas.getContext( | |
| "2d" | |
| ); | |
| context.drawImage( | |
| el.camera, | |
| 0, | |
| 0 | |
| ); | |
| /* | |
| * Also draw the latest detections | |
| * onto the snapshot. | |
| */ | |
| drawSnapshotDetections( | |
| context, | |
| canvas.width, | |
| canvas.height | |
| ); | |
| const link = | |
| document.createElement( | |
| "a" | |
| ); | |
| link.download = | |
| `locateanything-${Date.now()}.jpg`; | |
| link.href = | |
| canvas.toDataURL( | |
| "image/jpeg", | |
| 0.92 | |
| ); | |
| link.click(); | |
| } | |
| function drawSnapshotDetections( | |
| context, | |
| width, | |
| height | |
| ) { | |
| const sourceWidth = | |
| Number( | |
| el.camera.videoWidth | |
| ); | |
| const sourceHeight = | |
| Number( | |
| el.camera.videoHeight | |
| ); | |
| if ( | |
| !sourceWidth || | |
| !sourceHeight | |
| ) { | |
| return; | |
| } | |
| const scaleX = | |
| width / | |
| sourceWidth; | |
| const scaleY = | |
| height / | |
| sourceHeight; | |
| state.detections.forEach( | |
| ( | |
| detection, | |
| index | |
| ) => { | |
| const x = | |
| detection.x1 * | |
| scaleX; | |
| const y = | |
| detection.y1 * | |
| scaleY; | |
| const w = | |
| ( | |
| detection.x2 - | |
| detection.x1 | |
| ) * | |
| scaleX; | |
| const h = | |
| ( | |
| detection.y2 - | |
| detection.y1 | |
| ) * | |
| scaleY; | |
| const color = | |
| getDetectionColor( | |
| index | |
| ); | |
| context.strokeStyle = | |
| color; | |
| context.lineWidth = | |
| 4; | |
| context.strokeRect( | |
| x, | |
| y, | |
| w, | |
| h | |
| ); | |
| context.font = | |
| "bold 20px system-ui"; | |
| const label = | |
| detection.label || | |
| "object"; | |
| const textWidth = | |
| context.measureText( | |
| label | |
| ).width; | |
| context.fillStyle = | |
| "rgba(0,0,0,.85)"; | |
| context.fillRect( | |
| x, | |
| Math.max( | |
| 0, | |
| y - 32 | |
| ), | |
| textWidth + 20, | |
| 32 | |
| ); | |
| context.fillStyle = | |
| color; | |
| context.fillText( | |
| label, | |
| x + 10, | |
| Math.max( | |
| 22, | |
| y - 10 | |
| ) | |
| ); | |
| } | |
| ); | |
| } | |
| /* ========================================================= | |
| UPLOAD | |
| ========================================================= */ | |
| function handleUpload( | |
| event | |
| ) { | |
| const file = | |
| event.target.files && | |
| event.target.files[0]; | |
| if (!file) { | |
| return; | |
| } | |
| const url = | |
| URL.createObjectURL( | |
| file | |
| ); | |
| const image = | |
| new Image(); | |
| image.onload = () => { | |
| const canvas = | |
| document.createElement( | |
| "canvas" | |
| ); | |
| const maximum = | |
| 1024; | |
| const ratio = | |
| Math.min( | |
| 1, | |
| maximum / | |
| Math.max( | |
| image.width, | |
| image.height | |
| ) | |
| ); | |
| canvas.width = | |
| Math.round( | |
| image.width * | |
| ratio | |
| ); | |
| canvas.height = | |
| Math.round( | |
| image.height * | |
| ratio | |
| ); | |
| const context = | |
| canvas.getContext( | |
| "2d" | |
| ); | |
| context.drawImage( | |
| image, | |
| 0, | |
| 0, | |
| canvas.width, | |
| canvas.height | |
| ); | |
| const data = | |
| canvas.toDataURL( | |
| "image/jpeg", | |
| 0.75 | |
| ); | |
| sendImage( | |
| data | |
| ); | |
| URL.revokeObjectURL( | |
| url | |
| ); | |
| }; | |
| image.onerror = () => { | |
| URL.revokeObjectURL( | |
| url | |
| ); | |
| setStatus( | |
| "Upload failed", | |
| "The selected image could not be opened." | |
| ); | |
| }; | |
| image.src = | |
| url; | |
| event.target.value = | |
| ""; | |
| } | |
| /* ========================================================= | |
| SEND UPLOADED IMAGE | |
| ========================================================= */ | |
| function sendImage( | |
| image | |
| ) { | |
| if ( | |
| !state.socket || | |
| state.socket.readyState !== | |
| WebSocket.OPEN | |
| ) { | |
| connectSocket(); | |
| setStatus( | |
| "Connecting", | |
| "Connecting to inference server…" | |
| ); | |
| return; | |
| } | |
| if (state.processing) { | |
| setStatus( | |
| "Busy", | |
| "The previous image is still being analyzed." | |
| ); | |
| return; | |
| } | |
| state.frameId++; | |
| try { | |
| state.socket.send( | |
| JSON.stringify( | |
| { | |
| type: | |
| "frame", | |
| frame_id: | |
| state.frameId, | |
| image, | |
| query: | |
| state.query, | |
| mode: | |
| state.mode | |
| } | |
| ) | |
| ); | |
| state.processing = | |
| true; | |
| } catch (error) { | |
| console.error( | |
| error | |
| ); | |
| } | |
| } | |
| /* ========================================================= | |
| QUERY | |
| ========================================================= */ | |
| function setQuery( | |
| value | |
| ) { | |
| value = | |
| String( | |
| value || "" | |
| ) | |
| .trim() | |
| .substring( | |
| 0, | |
| 500 | |
| ); | |
| if (!value) { | |
| value = | |
| "all objects"; | |
| } | |
| state.query = | |
| value; | |
| if (el.query) { | |
| el.query.value = | |
| value; | |
| } | |
| if (el.chips) { | |
| el.chips.forEach( | |
| chip => { | |
| chip.classList.toggle( | |
| "active", | |
| chip.dataset.query === | |
| value | |
| ); | |
| } | |
| ); | |
| } | |
| if (state.running) { | |
| setStatus( | |
| "Target updated", | |
| `Searching for “${value}”.` | |
| ); | |
| } | |
| } | |
| /* ========================================================= | |
| MODE | |
| ========================================================= */ | |
| function setMode( | |
| mode | |
| ) { | |
| mode = | |
| String( | |
| mode || "hybrid" | |
| ) | |
| .toLowerCase(); | |
| if ( | |
| ![ | |
| "hybrid", | |
| "fast", | |
| "slow", | |
| ].includes(mode) | |
| ) { | |
| mode = | |
| "hybrid"; | |
| } | |
| state.mode = | |
| mode; | |
| if (el.modes) { | |
| el.modes.forEach( | |
| button => { | |
| button.classList.toggle( | |
| "active", | |
| button.dataset.mode === | |
| mode | |
| ); | |
| } | |
| ); | |
| } | |
| } | |
| /* ========================================================= | |
| RESULTS | |
| ========================================================= */ | |
| function updateResults( | |
| detections | |
| ) { | |
| text( | |
| el.resultsCount, | |
| String( | |
| detections.length | |
| ) | |
| ); | |
| if (!detections.length) { | |
| toggle( | |
| el.resultsPanel, | |
| "hidden", | |
| true | |
| ); | |
| if (el.resultsList) { | |
| el.resultsList.innerHTML = | |
| ""; | |
| } | |
| return; | |
| } | |
| toggle( | |
| el.resultsPanel, | |
| "hidden", | |
| false | |
| ); | |
| if (!el.resultsList) { | |
| return; | |
| } | |
| el.resultsList.innerHTML = | |
| detections | |
| .slice(0, 30) | |
| .map( | |
| ( | |
| detection, | |
| index | |
| ) => { | |
| const score = | |
| Number.isFinite( | |
| Number( | |
| detection.score | |
| ) | |
| ) | |
| ? ` ${ | |
| ( | |
| Number( | |
| detection.score | |
| ) * 100 | |
| ).toFixed(0) | |
| }%` | |
| : ""; | |
| return ` | |
| <div | |
| class="result-item" | |
| style=" | |
| animation-delay: | |
| ${index * 35}ms | |
| " | |
| > | |
| <span | |
| class="result-name" | |
| > | |
| ${escapeHTML( | |
| detection.label || | |
| "object" | |
| )} | |
| </span> | |
| <span | |
| class="result-index" | |
| > | |
| #${index + 1}${score} | |
| </span> | |
| </div> | |
| `; | |
| } | |
| ) | |
| .join(""); | |
| } | |
| /* ========================================================= | |
| HTML ESCAPE | |
| ========================================================= */ | |
| function escapeHTML( | |
| value | |
| ) { | |
| return String( | |
| value | |
| ) | |
| .replaceAll( | |
| "&", | |
| "&" | |
| ) | |
| .replaceAll( | |
| "<", | |
| "<" | |
| ) | |
| .replaceAll( | |
| ">", | |
| ">" | |
| ) | |
| .replaceAll( | |
| '"', | |
| """ | |
| ) | |
| .replaceAll( | |
| "'", | |
| "'" | |
| ); | |
| } | |
| /* ========================================================= | |
| OVERLAY | |
| ========================================================= */ | |
| function resizeCanvas() { | |
| if ( | |
| !el.overlay | |
| ) { | |
| return; | |
| } | |
| const rect = | |
| el.overlay.getBoundingClientRect(); | |
| const dpr = | |
| window.devicePixelRatio || | |
| 1; | |
| el.overlay.width = | |
| Math.max( | |
| 1, | |
| Math.round( | |
| rect.width * | |
| dpr | |
| ) | |
| ); | |
| el.overlay.height = | |
| Math.max( | |
| 1, | |
| Math.round( | |
| rect.height * | |
| dpr | |
| ) | |
| ); | |
| } | |
| function clearOverlay() { | |
| if (!el.overlay) { | |
| return; | |
| } | |
| const context = | |
| el.overlay.getContext( | |
| "2d" | |
| ); | |
| context.clearRect( | |
| 0, | |
| 0, | |
| el.overlay.width, | |
| el.overlay.height | |
| ); | |
| } | |
| function getVideoGeometry() { | |
| if ( | |
| !el.camera || | |
| !el.camera.videoWidth || | |
| !el.camera.videoHeight | |
| ) { | |
| return null; | |
| } | |
| const rect = | |
| el.camera.getBoundingClientRect(); | |
| return { | |
| width: | |
| el.camera.videoWidth, | |
| height: | |
| el.camera.videoHeight, | |
| displayWidth: | |
| rect.width, | |
| displayHeight: | |
| rect.height, | |
| left: | |
| rect.left, | |
| top: | |
| rect.top, | |
| }; | |
| } | |
| function drawOverlay() { | |
| if ( | |
| !el.overlay || | |
| !el.camera | |
| ) { | |
| return; | |
| } | |
| resizeCanvas(); | |
| const context = | |
| el.overlay.getContext( | |
| "2d" | |
| ); | |
| const dpr = | |
| window.devicePixelRatio || | |
| 1; | |
| context.clearRect( | |
| 0, | |
| 0, | |
| el.overlay.width, | |
| el.overlay.height | |
| ); | |
| const geometry = | |
| getVideoGeometry(); | |
| if (!geometry) { | |
| return; | |
| } | |
| context.save(); | |
| context.scale( | |
| dpr, | |
| dpr | |
| ); | |
| const displayWidth = | |
| el.overlay.clientWidth; | |
| const displayHeight = | |
| el.overlay.clientHeight; | |
| state.detections.forEach( | |
| ( | |
| detection, | |
| index | |
| ) => { | |
| /* | |
| * Backend coordinates are in the | |
| * captured image's pixel space. | |
| */ | |
| const x = | |
| detection.x1 / | |
| geometry.width * | |
| displayWidth; | |
| const y = | |
| detection.y1 / | |
| geometry.height * | |
| displayHeight; | |
| const width = | |
| ( | |
| detection.x2 - | |
| detection.x1 | |
| ) / | |
| geometry.width * | |
| displayWidth; | |
| const height = | |
| ( | |
| detection.y2 - | |
| detection.y1 | |
| ) / | |
| geometry.height * | |
| displayHeight; | |
| drawDetection( | |
| context, | |
| x, | |
| y, | |
| width, | |
| height, | |
| detection.label || | |
| "object", | |
| index | |
| ); | |
| } | |
| ); | |
| context.restore(); | |
| } | |
| function drawDetection( | |
| context, | |
| x, | |
| y, | |
| width, | |
| height, | |
| label, | |
| index | |
| ) { | |
| const color = | |
| getDetectionColor( | |
| index | |
| ); | |
| context.save(); | |
| context.strokeStyle = | |
| color; | |
| context.lineWidth = | |
| 2; | |
| context.shadowColor = | |
| color; | |
| context.shadowBlur = | |
| 10; | |
| roundRect( | |
| context, | |
| x, | |
| y, | |
| width, | |
| height, | |
| 8 | |
| ); | |
| context.stroke(); | |
| context.shadowBlur = | |
| 0; | |
| const fontSize = | |
| 13; | |
| context.font = | |
| `700 ${fontSize}px system-ui`; | |
| const textWidth = | |
| context.measureText( | |
| label | |
| ).width; | |
| const padding = | |
| 8; | |
| const labelWidth = | |
| textWidth + | |
| padding * 2; | |
| const labelHeight = | |
| 30; | |
| let labelX = | |
| x; | |
| let labelY = | |
| y - | |
| labelHeight - | |
| 5; | |
| if ( | |
| labelY < 2 | |
| ) { | |
| labelY = | |
| y + 5; | |
| } | |
| context.fillStyle = | |
| "rgba(3, 7, 8, 0.90)"; | |
| roundRect( | |
| context, | |
| labelX, | |
| labelY, | |
| labelWidth, | |
| labelHeight, | |
| 7 | |
| ); | |
| context.fill(); | |
| context.fillStyle = | |
| color; | |
| context.textBaseline = | |
| "middle"; | |
| context.fillText( | |
| label, | |
| labelX + | |
| padding, | |
| labelY + | |
| labelHeight / 2 | |
| ); | |
| context.restore(); | |
| } | |
| function roundRect( | |
| context, | |
| x, | |
| y, | |
| width, | |
| height, | |
| radius | |
| ) { | |
| radius = | |
| Math.min( | |
| radius, | |
| width / 2, | |
| height / 2 | |
| ); | |
| context.beginPath(); | |
| context.moveTo( | |
| x + radius, | |
| y | |
| ); | |
| context.arcTo( | |
| x + width, | |
| y, | |
| x + width, | |
| y + height, | |
| radius | |
| ); | |
| context.arcTo( | |
| x + width, | |
| y + height, | |
| x, | |
| y + height, | |
| radius | |
| ); | |
| context.arcTo( | |
| x, | |
| y + height, | |
| x, | |
| y, | |
| radius | |
| ); | |
| context.arcTo( | |
| x, | |
| y, | |
| x + width, | |
| y, | |
| radius | |
| ); | |
| context.closePath(); | |
| } | |
| function getDetectionColor( | |
| index | |
| ) { | |
| const colors = [ | |
| "#8bf0b9", | |
| "#82c7ff", | |
| "#ffc56b", | |
| "#c79aff", | |
| "#ff8cae", | |
| ]; | |
| return colors[ | |
| index % | |
| colors.length | |
| ]; | |
| } | |
| /* ========================================================= | |
| FPS | |
| ========================================================= */ | |
| function startCameraFPS() { | |
| state.fpsFrames = | |
| 0; | |
| state.fpsStarted = | |
| performance.now(); | |
| requestAnimationFrame( | |
| cameraFPSLoop | |
| ); | |
| } | |
| function cameraFPSLoop() { | |
| if (!state.running) { | |
| return; | |
| } | |
| state.fpsFrames++; | |
| const now = | |
| performance.now(); | |
| const elapsed = | |
| now - | |
| state.fpsStarted; | |
| if ( | |
| elapsed >= 1000 | |
| ) { | |
| const fps = | |
| state.fpsFrames / | |
| ( | |
| elapsed / | |
| 1000 | |
| ); | |
| text( | |
| el.fpsBadge, | |
| `${fps.toFixed(0)} FPS` | |
| ); | |
| state.fpsFrames = | |
| 0; | |
| state.fpsStarted = | |
| now; | |
| } | |
| requestAnimationFrame( | |
| cameraFPSLoop | |
| ); | |
| } | |
| function updateInferenceFPS() { | |
| const now = | |
| performance.now(); | |
| state.inferenceCount++; | |
| const elapsed = | |
| now - | |
| state.inferenceFpsStarted; | |
| if ( | |
| elapsed >= 1000 | |
| ) { | |
| const fps = | |
| state.inferenceCount / | |
| ( | |
| elapsed / | |
| 1000 | |
| ); | |
| text( | |
| el.inferenceFps, | |
| `${fps.toFixed(2)} FPS` | |
| ); | |
| state.inferenceCount = | |
| 0; | |
| state.inferenceFpsStarted = | |
| now; | |
| } | |
| } | |
| /* ========================================================= | |
| EVENTS | |
| ========================================================= */ | |
| if (el.startButton) { | |
| el.startButton.addEventListener( | |
| "click", | |
| startCamera | |
| ); | |
| } | |
| if (el.stopButton) { | |
| el.stopButton.addEventListener( | |
| "click", | |
| stopCamera | |
| ); | |
| } | |
| if (el.mobileStart) { | |
| el.mobileStart.addEventListener( | |
| "click", | |
| startCamera | |
| ); | |
| } | |
| if (el.snapshotButton) { | |
| el.snapshotButton.addEventListener( | |
| "click", | |
| snapshot | |
| ); | |
| } | |
| if (el.mobileSnapshot) { | |
| el.mobileSnapshot.addEventListener( | |
| "click", | |
| snapshot | |
| ); | |
| } | |
| if (el.permissionClose) { | |
| el.permissionClose.addEventListener( | |
| "click", | |
| () => { | |
| toggle( | |
| el.permissionModal, | |
| "hidden", | |
| true | |
| ); | |
| startCamera(); | |
| } | |
| ); | |
| } | |
| if (el.clearQuery) { | |
| el.clearQuery.addEventListener( | |
| "click", | |
| () => { | |
| if (el.query) { | |
| el.query.value = | |
| ""; | |
| el.query.focus(); | |
| } | |
| state.query = | |
| "all objects"; | |
| } | |
| ); | |
| } | |
| if (el.query) { | |
| el.query.addEventListener( | |
| "input", | |
| event => { | |
| state.query = | |
| event.target.value | |
| .trim() | |
| .substring( | |
| 0, | |
| 500 | |
| ); | |
| } | |
| ); | |
| el.query.addEventListener( | |
| "keydown", | |
| event => { | |
| if ( | |
| event.key === | |
| "Enter" | |
| ) { | |
| setQuery( | |
| event.target.value | |
| ); | |
| } | |
| } | |
| ); | |
| } | |
| if (el.chips) { | |
| el.chips.forEach( | |
| chip => { | |
| chip.addEventListener( | |
| "click", | |
| () => { | |
| setQuery( | |
| chip.dataset.query | |
| ); | |
| } | |
| ); | |
| } | |
| ); | |
| } | |
| if (el.modes) { | |
| el.modes.forEach( | |
| button => { | |
| button.addEventListener( | |
| "click", | |
| () => { | |
| setMode( | |
| button.dataset.mode | |
| ); | |
| } | |
| ); | |
| } | |
| ); | |
| } | |
| if (el.imageUpload) { | |
| el.imageUpload.addEventListener( | |
| "change", | |
| handleUpload | |
| ); | |
| } | |
| window.addEventListener( | |
| "resize", | |
| () => { | |
| resizeCanvas(); | |
| drawOverlay(); | |
| } | |
| ); | |
| if (el.camera) { | |
| el.camera.addEventListener( | |
| "loadedmetadata", | |
| () => { | |
| resizeCanvas(); | |
| drawOverlay(); | |
| } | |
| ); | |
| } | |
| /* ========================================================= | |
| INITIALIZE | |
| ========================================================= */ | |
| console.log( | |
| "LocateAnything Live V2 initializing..." | |
| ); | |
| setConnection( | |
| "offline", | |
| "Connecting" | |
| ); | |
| setQuery( | |
| "person" | |
| ); | |
| setMode( | |
| "hybrid" | |
| ); | |
| updateButtons(); | |
| connectSocket(); | |
| console.log( | |
| "LocateAnything Live V2 ready." | |
| ); |