"use strict"; /* ============================================================ LocateAnything Live V2 Smooth Camera + Async CPU Inference ============================================================ */ const state = { stream: null, running: false, connected: false, processing: false, currentJob: null, frameId: 0, query: "person", mode: "hybrid", pollTimer: null, captureTimer: null, reconnectTimer: null, captureInterval: 1200, detections: [], inferenceStarted: 0, fpsFrames: 0, fpsStarted: performance.now(), inferenceCount: 0, inferenceFpsStarted: performance.now(), }; /* ========================================================= ELEMENTS ========================================================= */ const el = { camera: document.getElementById("camera"), overlay: document.getElementById("overlay"), cameraPlaceholder: document.getElementById( "cameraPlaceholder" ), startButton: document.getElementById( "startButton" ), stopButton: document.getElementById( "stopButton" ), snapshotButton: document.getElementById( "snapshotButton" ), mobileStart: document.getElementById( "mobileStart" ), mobileSnapshot: document.getElementById( "mobileSnapshot" ), imageUpload: document.getElementById( "imageUpload" ), query: document.getElementById( "query" ), clearQuery: document.getElementById( "clearQuery" ), chips: document.querySelectorAll( ".chip" ), modes: document.querySelectorAll( ".mode-button" ), connectionDot: document.getElementById( "connectionDot" ), connectionText: document.getElementById( "connectionText" ), liveBadge: document.getElementById( "liveBadge" ), fpsBadge: document.getElementById( "fpsBadge" ), reticle: document.getElementById( "reticle" ), objectCount: document.getElementById( "objectCount" ), latency: document.getElementById( "latency" ), inferenceFps: document.getElementById( "inferenceFps" ), statusTitle: document.getElementById( "statusTitle" ), statusMessage: document.getElementById( "statusMessage" ), resultsPanel: document.getElementById( "resultsPanel" ), resultsCount: document.getElementById( "resultsCount" ), resultsList: document.getElementById( "resultsList" ), permissionModal: document.getElementById( "permissionModal" ), permissionClose: document.getElementById( "permissionClose" ), }; /* ========================================================= SAFE ELEMENT HELPERS ========================================================= */ function exists(element) { return !!element; } function text(element, value) { if (element) { element.textContent = value; } } function toggle( element, className, enabled ) { if (!element) { return; } element.classList.toggle( className, enabled ); } /* ========================================================= STATUS ========================================================= */ function setConnection( status, message ) { state.connected = status === "online"; if (el.connectionDot) { el.connectionDot.className = `connection-dot ${status}`; } text( el.connectionText, message ); } function setStatus( title, message ) { text( el.statusTitle, title ); text( el.statusMessage, message ); } function setLive( enabled ) { toggle( el.liveBadge, "hidden", !enabled ); toggle( el.fpsBadge, "hidden", !enabled ); toggle( el.reticle, "hidden", !enabled ); } function updateButtons() { if (el.startButton) { el.startButton.disabled = state.running; } if (el.stopButton) { el.stopButton.disabled = !state.running; } if (el.snapshotButton) { el.snapshotButton.disabled = !state.running; } if (el.mobileSnapshot) { el.mobileSnapshot.disabled = !state.running; } if (el.mobileStart) { el.mobileStart.textContent = state.running ? "Stop" : "Start"; } } /* ========================================================= WEBSOCKET ========================================================= */ function websocketURL() { const protocol = location.protocol === "https:" ? "wss:" : "ws:"; return ( `${protocol}//${location.host}/ws` ); } function connectSocket() { if ( state.socket && ( state.socket.readyState === WebSocket.OPEN || state.socket.readyState === WebSocket.CONNECTING ) ) { return; } setConnection( "offline", "Connecting" ); try { state.socket = new WebSocket( websocketURL() ); } catch (error) { console.error(error); scheduleReconnect(); return; } state.socket.onopen = () => { console.log( "LocateAnything WebSocket connected." ); setConnection( "online", "Connected" ); if (state.running) { setStatus( "Camera active", `Searching for “${state.query}”.` ); } else { setStatus( "Ready to locate", "Start the camera to begin." ); } }; state.socket.onmessage = ( event ) => { handleServerMessage( event.data ); }; state.socket.onerror = ( error ) => { console.error( "WebSocket error:", error ); setConnection( "error", "Connection error" ); }; state.socket.onclose = () => { state.connected = false; setConnection( "offline", "Disconnected" ); if (state.running) { setStatus( "Reconnecting", "Reconnecting to the server…" ); scheduleReconnect(); } }; } function scheduleReconnect() { if (state.reconnectTimer) { return; } state.reconnectTimer = setTimeout( () => { state.reconnectTimer = null; connectSocket(); }, 1500 ); } /* ========================================================= SERVER MESSAGES ========================================================= */ function handleServerMessage( raw ) { let message; try { message = JSON.parse(raw); } catch (error) { console.error( "Invalid server JSON:", raw ); return; } console.log( "LocateAnything:", message ); switch ( message.type ) { case "connected": state.connected = true; setConnection( "online", "Connected" ); break; case "job": handleJobCreated( message ); break; case "busy": handleBusy( message ); break; case "processing": setStatus( "Analyzing", `Locating “${state.query}”…` ); break; case "result": /* * Compatibility with the older backend. */ handleResult( message ); break; case "error": handleError( message ); break; case "stopped": state.processing = false; state.currentJob = null; break; case "pong": break; default: console.log( "Unknown server message:", message ); } } /* ========================================================= JOB CREATED ========================================================= */ function handleJobCreated( message ) { const jobId = message.job_id; if (!jobId) { state.processing = false; setStatus( "Job error", "The server did not return a job ID." ); return; } state.currentJob = jobId; state.processing = true; state.inferenceStarted = performance.now(); setStatus( "Analyzing", `Locating “${state.query}”…` ); /* * Start polling. * * The camera itself continues running. */ pollJob( jobId ); } /* ========================================================= POLL JOB ========================================================= */ async function pollJob( jobId ) { if ( !state.processing || state.currentJob !== jobId ) { return; } try { const response = await fetch( `/job/${encodeURIComponent( jobId )}`, { method: "GET", cache: "no-store", } ); if (!response.ok) { throw new Error( `HTTP ${response.status}` ); } const job = await response.json(); if ( job.status === "queued" ) { updateJobTimer(); scheduleJobPoll( jobId, 1500 ); return; } if ( job.status === "running" ) { updateJobTimer(); scheduleJobPoll( jobId, 2000 ); return; } if ( job.status === "completed" ) { state.processing = false; state.currentJob = null; handleResult( job ); /* * Give the camera a moment before * submitting the next frame. */ scheduleCapture( 800 ); return; } if ( job.status === "error" ) { state.processing = false; state.currentJob = null; handleError( job ); scheduleCapture( 1500 ); return; } scheduleJobPoll( jobId, 1500 ); } catch (error) { console.error( "Job polling error:", error ); /* * Don't immediately kill the job. * * The CPU inference can still be running. */ scheduleJobPoll( jobId, 3000 ); } } function scheduleJobPoll( jobId, delay ) { if (state.pollTimer) { clearTimeout( state.pollTimer ); } state.pollTimer = setTimeout( () => { state.pollTimer = null; pollJob( jobId ); }, delay ); } /* ========================================================= INFERENCE TIMER ========================================================= */ function updateJobTimer() { if ( !state.inferenceStarted ) { return; } const elapsed = performance.now() - state.inferenceStarted; const seconds = elapsed / 1000; setStatus( "Analyzing", `Locating “${state.query}”… ${seconds.toFixed(0)}s` ); } /* ========================================================= RESULT ========================================================= */ function handleResult( message ) { const detections = Array.isArray( message.detections ) ? message.detections : []; state.detections = detections; state.inferenceCount++; text( el.objectCount, String( detections.length ) ); if ( Number.isFinite( Number( message.latency ) ) ) { text( el.latency, `${( Number( message.latency ) / 1000 ).toFixed(1)}s` ); } updateInferenceFPS(); updateResults( detections ); drawOverlay(); if ( detections.length ) { setStatus( `${detections.length} ${ detections.length === 1 ? "object" : "objects" } located`, `Latest result for “${state.query}”.` ); } else { setStatus( "Nothing located", `No matching “${state.query}” was found in the latest frame.` ); } } /* ========================================================= ERROR ========================================================= */ function handleError( message ) { console.error( "LocateAnything error:", message ); setStatus( "Inference error", message.detail || message.message || "Inference failed." ); } /* ========================================================= BUSY ========================================================= */ function handleBusy( message ) { state.processing = true; setStatus( "Analyzing", message.message || "Previous inference is still running." ); } /* ========================================================= CAMERA ========================================================= */ async function startCamera() { if (state.running) { stopCamera(); return; } try { if ( !navigator.mediaDevices || !navigator.mediaDevices .getUserMedia ) { throw new Error( "Camera API is unavailable." ); } const stream = await navigator .mediaDevices .getUserMedia( { video: { facingMode: { ideal: "environment" }, width: { ideal: 1280 }, height: { ideal: 720 }, frameRate: { ideal: 30, max: 30 } }, audio: false } ); state.stream = stream; el.camera.srcObject = stream; await el.camera.play(); state.running = true; state.processing = false; if ( el.cameraPlaceholder ) { el.cameraPlaceholder .classList .add( "hidden" ); } setLive( true ); updateButtons(); setStatus( "Camera active", `Searching for “${state.query}”.` ); connectSocket(); startCameraFPS(); resizeCanvas(); /* * Submit the first frame shortly * after the camera starts. */ scheduleCapture( 1000 ); } catch (error) { console.error( "Camera error:", error ); if ( el.permissionModal ) { el.permissionModal .classList .remove( "hidden" ); } setStatus( "Camera unavailable", error.message || "Please allow camera access." ); } } /* ========================================================= STOP ========================================================= */ function stopCamera() { state.running = false; state.processing = false; state.currentJob = null; if ( state.captureTimer ) { clearTimeout( state.captureTimer ); state.captureTimer = null; } if ( state.pollTimer ) { clearTimeout( state.pollTimer ); state.pollTimer = null; } if ( state.stream ) { state.stream .getTracks() .forEach( track => track.stop() ); state.stream = null; } if ( el.camera ) { el.camera.srcObject = null; } state.detections = []; clearOverlay(); setLive( false ); updateButtons(); if ( el.cameraPlaceholder ) { el.cameraPlaceholder .classList .remove( "hidden" ); } setStatus( "Camera stopped", "Start the camera to continue." ); if ( state.socket && state.socket.readyState === WebSocket.OPEN ) { try { state.socket.send( JSON.stringify( { type: "stop" } ) ); } catch (error) { console.warn( error ); } } } /* ========================================================= CAMERA CAPTURE ========================================================= */ function scheduleCapture( delay ) { if (!state.running) { return; } if ( state.captureTimer ) { clearTimeout( state.captureTimer ); } state.captureTimer = setTimeout( () => { state.captureTimer = null; captureFrame(); }, delay ); } function captureFrame() { if (!state.running) { return; } /* * Never submit another frame while * the previous CPU inference is running. */ if (state.processing) { return; } if ( !state.socket || state.socket.readyState !== WebSocket.OPEN ) { connectSocket(); scheduleCapture( 2000 ); return; } const video = el.camera; if ( !video.videoWidth || !video.videoHeight ) { scheduleCapture( 1000 ); return; } const canvas = document.createElement( "canvas" ); const maximum = 768; const ratio = Math.min( 1, maximum / Math.max( video.videoWidth, video.videoHeight ) ); canvas.width = Math.round( video.videoWidth * ratio ); canvas.height = Math.round( video.videoHeight * ratio ); const context = canvas.getContext( "2d", { alpha: false } ); context.drawImage( video, 0, 0, canvas.width, canvas.height ); const image = canvas.toDataURL( "image/jpeg", 0.68 ); state.frameId++; try { state.socket.send( JSON.stringify( { type: "frame", frame_id: state.frameId, image, query: state.query, mode: state.mode } ) ); /* * The server will send "job". * * Do NOT schedule another capture * here. The next frame is scheduled * after the current job completes. */ } catch (error) { console.error( "Frame send failed:", error ); scheduleCapture( 2000 ); } } /* ========================================================= SNAPSHOT ========================================================= */ function snapshot() { if ( !state.running || !el.camera.videoWidth ) { return; } const canvas = document.createElement( "canvas" ); canvas.width = el.camera.videoWidth; canvas.height = el.camera.videoHeight; const context = canvas.getContext( "2d" ); context.drawImage( el.camera, 0, 0 ); /* * Also draw the latest detections * onto the snapshot. */ drawSnapshotDetections( context, canvas.width, canvas.height ); const link = document.createElement( "a" ); link.download = `locateanything-${Date.now()}.jpg`; link.href = canvas.toDataURL( "image/jpeg", 0.92 ); link.click(); } function drawSnapshotDetections( context, width, height ) { const sourceWidth = Number( el.camera.videoWidth ); const sourceHeight = Number( el.camera.videoHeight ); if ( !sourceWidth || !sourceHeight ) { return; } const scaleX = width / sourceWidth; const scaleY = height / sourceHeight; state.detections.forEach( ( detection, index ) => { const x = detection.x1 * scaleX; const y = detection.y1 * scaleY; const w = ( detection.x2 - detection.x1 ) * scaleX; const h = ( detection.y2 - detection.y1 ) * scaleY; const color = getDetectionColor( index ); context.strokeStyle = color; context.lineWidth = 4; context.strokeRect( x, y, w, h ); context.font = "bold 20px system-ui"; const label = detection.label || "object"; const textWidth = context.measureText( label ).width; context.fillStyle = "rgba(0,0,0,.85)"; context.fillRect( x, Math.max( 0, y - 32 ), textWidth + 20, 32 ); context.fillStyle = color; context.fillText( label, x + 10, Math.max( 22, y - 10 ) ); } ); } /* ========================================================= UPLOAD ========================================================= */ function handleUpload( event ) { const file = event.target.files && event.target.files[0]; if (!file) { return; } const url = URL.createObjectURL( file ); const image = new Image(); image.onload = () => { const canvas = document.createElement( "canvas" ); const maximum = 1024; const ratio = Math.min( 1, maximum / Math.max( image.width, image.height ) ); canvas.width = Math.round( image.width * ratio ); canvas.height = Math.round( image.height * ratio ); const context = canvas.getContext( "2d" ); context.drawImage( image, 0, 0, canvas.width, canvas.height ); const data = canvas.toDataURL( "image/jpeg", 0.75 ); sendImage( data ); URL.revokeObjectURL( url ); }; image.onerror = () => { URL.revokeObjectURL( url ); setStatus( "Upload failed", "The selected image could not be opened." ); }; image.src = url; event.target.value = ""; } /* ========================================================= SEND UPLOADED IMAGE ========================================================= */ function sendImage( image ) { if ( !state.socket || state.socket.readyState !== WebSocket.OPEN ) { connectSocket(); setStatus( "Connecting", "Connecting to inference server…" ); return; } if (state.processing) { setStatus( "Busy", "The previous image is still being analyzed." ); return; } state.frameId++; try { state.socket.send( JSON.stringify( { type: "frame", frame_id: state.frameId, image, query: state.query, mode: state.mode } ) ); state.processing = true; } catch (error) { console.error( error ); } } /* ========================================================= QUERY ========================================================= */ function setQuery( value ) { value = String( value || "" ) .trim() .substring( 0, 500 ); if (!value) { value = "all objects"; } state.query = value; if (el.query) { el.query.value = value; } if (el.chips) { el.chips.forEach( chip => { chip.classList.toggle( "active", chip.dataset.query === value ); } ); } if (state.running) { setStatus( "Target updated", `Searching for “${value}”.` ); } } /* ========================================================= MODE ========================================================= */ function setMode( mode ) { mode = String( mode || "hybrid" ) .toLowerCase(); if ( ![ "hybrid", "fast", "slow", ].includes(mode) ) { mode = "hybrid"; } state.mode = mode; if (el.modes) { el.modes.forEach( button => { button.classList.toggle( "active", button.dataset.mode === mode ); } ); } } /* ========================================================= RESULTS ========================================================= */ function updateResults( detections ) { text( el.resultsCount, String( detections.length ) ); if (!detections.length) { toggle( el.resultsPanel, "hidden", true ); if (el.resultsList) { el.resultsList.innerHTML = ""; } return; } toggle( el.resultsPanel, "hidden", false ); if (!el.resultsList) { return; } el.resultsList.innerHTML = detections .slice(0, 30) .map( ( detection, index ) => { const score = Number.isFinite( Number( detection.score ) ) ? ` ${ ( Number( detection.score ) * 100 ).toFixed(0) }%` : ""; return `
${escapeHTML( detection.label || "object" )} #${index + 1}${score}
`; } ) .join(""); } /* ========================================================= HTML ESCAPE ========================================================= */ function escapeHTML( value ) { return String( value ) .replaceAll( "&", "&" ) .replaceAll( "<", "<" ) .replaceAll( ">", ">" ) .replaceAll( '"', """ ) .replaceAll( "'", "'" ); } /* ========================================================= OVERLAY ========================================================= */ function resizeCanvas() { if ( !el.overlay ) { return; } const rect = el.overlay.getBoundingClientRect(); const dpr = window.devicePixelRatio || 1; el.overlay.width = Math.max( 1, Math.round( rect.width * dpr ) ); el.overlay.height = Math.max( 1, Math.round( rect.height * dpr ) ); } function clearOverlay() { if (!el.overlay) { return; } const context = el.overlay.getContext( "2d" ); context.clearRect( 0, 0, el.overlay.width, el.overlay.height ); } function getVideoGeometry() { if ( !el.camera || !el.camera.videoWidth || !el.camera.videoHeight ) { return null; } const rect = el.camera.getBoundingClientRect(); return { width: el.camera.videoWidth, height: el.camera.videoHeight, displayWidth: rect.width, displayHeight: rect.height, left: rect.left, top: rect.top, }; } function drawOverlay() { if ( !el.overlay || !el.camera ) { return; } resizeCanvas(); const context = el.overlay.getContext( "2d" ); const dpr = window.devicePixelRatio || 1; context.clearRect( 0, 0, el.overlay.width, el.overlay.height ); const geometry = getVideoGeometry(); if (!geometry) { return; } context.save(); context.scale( dpr, dpr ); const displayWidth = el.overlay.clientWidth; const displayHeight = el.overlay.clientHeight; state.detections.forEach( ( detection, index ) => { /* * Backend coordinates are in the * captured image's pixel space. */ const x = detection.x1 / geometry.width * displayWidth; const y = detection.y1 / geometry.height * displayHeight; const width = ( detection.x2 - detection.x1 ) / geometry.width * displayWidth; const height = ( detection.y2 - detection.y1 ) / geometry.height * displayHeight; drawDetection( context, x, y, width, height, detection.label || "object", index ); } ); context.restore(); } function drawDetection( context, x, y, width, height, label, index ) { const color = getDetectionColor( index ); context.save(); context.strokeStyle = color; context.lineWidth = 2; context.shadowColor = color; context.shadowBlur = 10; roundRect( context, x, y, width, height, 8 ); context.stroke(); context.shadowBlur = 0; const fontSize = 13; context.font = `700 ${fontSize}px system-ui`; const textWidth = context.measureText( label ).width; const padding = 8; const labelWidth = textWidth + padding * 2; const labelHeight = 30; let labelX = x; let labelY = y - labelHeight - 5; if ( labelY < 2 ) { labelY = y + 5; } context.fillStyle = "rgba(3, 7, 8, 0.90)"; roundRect( context, labelX, labelY, labelWidth, labelHeight, 7 ); context.fill(); context.fillStyle = color; context.textBaseline = "middle"; context.fillText( label, labelX + padding, labelY + labelHeight / 2 ); context.restore(); } function roundRect( context, x, y, width, height, radius ) { radius = Math.min( radius, width / 2, height / 2 ); context.beginPath(); context.moveTo( x + radius, y ); context.arcTo( x + width, y, x + width, y + height, radius ); context.arcTo( x + width, y + height, x, y + height, radius ); context.arcTo( x, y + height, x, y, radius ); context.arcTo( x, y, x + width, y, radius ); context.closePath(); } function getDetectionColor( index ) { const colors = [ "#8bf0b9", "#82c7ff", "#ffc56b", "#c79aff", "#ff8cae", ]; return colors[ index % colors.length ]; } /* ========================================================= FPS ========================================================= */ function startCameraFPS() { state.fpsFrames = 0; state.fpsStarted = performance.now(); requestAnimationFrame( cameraFPSLoop ); } function cameraFPSLoop() { if (!state.running) { return; } state.fpsFrames++; const now = performance.now(); const elapsed = now - state.fpsStarted; if ( elapsed >= 1000 ) { const fps = state.fpsFrames / ( elapsed / 1000 ); text( el.fpsBadge, `${fps.toFixed(0)} FPS` ); state.fpsFrames = 0; state.fpsStarted = now; } requestAnimationFrame( cameraFPSLoop ); } function updateInferenceFPS() { const now = performance.now(); state.inferenceCount++; const elapsed = now - state.inferenceFpsStarted; if ( elapsed >= 1000 ) { const fps = state.inferenceCount / ( elapsed / 1000 ); text( el.inferenceFps, `${fps.toFixed(2)} FPS` ); state.inferenceCount = 0; state.inferenceFpsStarted = now; } } /* ========================================================= EVENTS ========================================================= */ if (el.startButton) { el.startButton.addEventListener( "click", startCamera ); } if (el.stopButton) { el.stopButton.addEventListener( "click", stopCamera ); } if (el.mobileStart) { el.mobileStart.addEventListener( "click", startCamera ); } if (el.snapshotButton) { el.snapshotButton.addEventListener( "click", snapshot ); } if (el.mobileSnapshot) { el.mobileSnapshot.addEventListener( "click", snapshot ); } if (el.permissionClose) { el.permissionClose.addEventListener( "click", () => { toggle( el.permissionModal, "hidden", true ); startCamera(); } ); } if (el.clearQuery) { el.clearQuery.addEventListener( "click", () => { if (el.query) { el.query.value = ""; el.query.focus(); } state.query = "all objects"; } ); } if (el.query) { el.query.addEventListener( "input", event => { state.query = event.target.value .trim() .substring( 0, 500 ); } ); el.query.addEventListener( "keydown", event => { if ( event.key === "Enter" ) { setQuery( event.target.value ); } } ); } if (el.chips) { el.chips.forEach( chip => { chip.addEventListener( "click", () => { setQuery( chip.dataset.query ); } ); } ); } if (el.modes) { el.modes.forEach( button => { button.addEventListener( "click", () => { setMode( button.dataset.mode ); } ); } ); } if (el.imageUpload) { el.imageUpload.addEventListener( "change", handleUpload ); } window.addEventListener( "resize", () => { resizeCanvas(); drawOverlay(); } ); if (el.camera) { el.camera.addEventListener( "loadedmetadata", () => { resizeCanvas(); drawOverlay(); } ); } /* ========================================================= INITIALIZE ========================================================= */ console.log( "LocateAnything Live V2 initializing..." ); setConnection( "offline", "Connecting" ); setQuery( "person" ); setMode( "hybrid" ); updateButtons(); connectSocket(); console.log( "LocateAnything Live V2 ready." );