using System; using UnityEngine; namespace OnDeviceAgent.Inference { public sealed class CameraInputService : MonoBehaviour { [SerializeField] bool m_StartOnEnable = true; [SerializeField, Tooltip("Camera device name; empty = system default.")] string m_DeviceName = ""; [SerializeField] int m_RequestedWidth = 1280; [SerializeField] int m_RequestedHeight = 720; [SerializeField] int m_RequestedFps = 30; [SerializeField, Range(10, 100)] int m_JpegQuality = 75; [SerializeField, Tooltip("Auto-advance to the next camera if the current one delivers no frames within this many seconds. Headless devices (no preview to tap) need this when device 0 is a non-streaming sensor.")] float m_ProbeTimeoutSeconds = 3f; [SerializeField, Tooltip("If no camera is streaming (e.g. the HAL was wedged by a previous run), re-probe every this many seconds so it recovers without an app restart.")] float m_CameraRetrySeconds = 8f; [SerializeField, Tooltip("Wait this long after releasing a camera before opening the next one, so the HAL frees the previous client first.")] float m_OpenCooldownSeconds = 0.7f; WebCamTexture m_WebCam; Texture2D m_ReadbackTex; int m_DeviceIndex = -1; int m_PendingDeviceIndex = -1; float m_OpenAt; bool m_Probing; int m_ProbeAttempts; float m_ProbeDeadline; float m_NextProbeLog; float m_LastUpdateTime; float m_NextRetryAt; RenderTexture m_PumpRt; bool m_LoggedOrientation; public bool IsRunning => m_WebCam != null && m_WebCam.isPlaying; public bool HasFrame => IsRunning && m_WebCam.width > 16; public string CurrentDeviceName => m_WebCam != null ? m_WebCam.deviceName : null; void OnEnable() { if (m_StartOnEnable) StartCamera(); } void OnDisable() { StopCamera(); } // Headless probe: WebCamTexture.Play() succeeds even on sensors that never deliver frames (width stays // at 16). With no preview to tap, auto-advance through the devices until one actually streams. void Update() { var now = Time.realtimeSinceStartup; var dt = m_LastUpdateTime > 0f ? now - m_LastUpdateTime : 0f; m_LastUpdateTime = now; // A device switch is pending: we stopped the previous camera and wait out a cooldown so the HAL // releases that client before opening the next. Opening too soon piles up clients until the service // rejects every open ("Too many other clients connecting") and no device ever streams. if (m_PendingDeviceIndex >= 0) { if (now >= m_OpenAt) OpenPendingDevice(); return; } // Android only pumps the camera SurfaceTexture (UpdateTexImage) when the texture is sampled. With the // preview removed nothing renders it, so frames never arrive. Blit it off-screen every frame to force // the update even when there is no visible consumer. if (m_WebCam != null && m_WebCam.isPlaying) { if (m_PumpRt == null) m_PumpRt = new RenderTexture(64, 64, 0); Graphics.Blit(m_WebCam, m_PumpRt); } if (!m_Probing) { MaybeRetryCamera(now); return; } if (m_WebCam != null && m_WebCam.isPlaying && m_WebCam.width > 16) { m_Probing = false; m_NextRetryAt = 0f; Debug.Log($"[Vision] Camera streaming: {m_WebCam.deviceName} ({m_WebCam.width}x{m_WebCam.height})"); return; } // A long stall (e.g. the ~10s on-device model load blocks the main thread) must not count against the // probe window: the camera can't pump while Update isn't ticking, so push the deadline past the stall. // Otherwise the deadline expires during the freeze and we thrash every device with rapid Stop/Play, // which wedges the camera HAL so none recover. if (dt > 0.75f) m_ProbeDeadline += dt; if (now >= m_NextProbeLog) { m_NextProbeLog = now + 1f; Debug.Log($"[Vision] probe {CurrentDeviceName}: {(m_WebCam != null ? m_WebCam.width : -1)}x{(m_WebCam != null ? m_WebCam.height : -1)} isPlaying={(m_WebCam != null && m_WebCam.isPlaying)} didUpdate={(m_WebCam != null && m_WebCam.didUpdateThisFrame)}"); } if (now < m_ProbeDeadline) return; var devices = WebCamTexture.devices; m_ProbeAttempts++; if (m_ProbeAttempts >= devices.Length) { // Tried every device once and none streamed. Stop cycling and let MaybeRetryCamera re-probe on a // slow cadence (the HAL is often just wedged and recovers) instead of thrashing Stop/Play. Debug.LogWarning($"[Vision] No camera delivered frames after trying {devices.Length} device(s); re-probing in {m_CameraRetrySeconds}s."); m_Probing = false; m_NextRetryAt = 0f; // arm MaybeRetryCamera return; } Debug.Log($"[Vision] Camera '{CurrentDeviceName}' delivered no frames in {m_ProbeTimeoutSeconds}s; trying next device."); StartDeviceAt((m_DeviceIndex + 1) % devices.Length); } // After the probe gives up, the camera may still come back (the HAL is often just wedged by a previous // run). Re-probe on a slow cadence so it recovers without an app restart, instead of staying dark forever. void MaybeRetryCamera(float now) { if (HasFrame) { m_NextRetryAt = 0f; return; } if (m_NextRetryAt <= 0f) { m_NextRetryAt = now + m_CameraRetrySeconds; // arm the wait return; } if (now < m_NextRetryAt) return; m_NextRetryAt = 0f; // consumed; re-armed once this re-probe gives up again Debug.Log("[Vision] Camera not streaming; re-probing devices."); StartCamera(); } public void StartCamera() { var devices = WebCamTexture.devices; if (devices.Length == 0) { Debug.LogWarning("[Vision] No camera device found."); return; } if (m_DeviceIndex < 0 || m_DeviceIndex >= devices.Length) m_DeviceIndex = ResolveInitialIndex(devices); m_ProbeAttempts = 0; StartDeviceAt(m_DeviceIndex); } // Releases the current camera and schedules the open of `index` after the cooldown; the actual open // happens in OpenPendingDevice (driven from Update) so we never hold two camera clients at once. void StartDeviceAt(int index) { var devices = WebCamTexture.devices; if (devices.Length == 0) return; index = ((index % devices.Length) + devices.Length) % devices.Length; StopCamera(); m_Probing = false; // not probing until the device actually opens m_PendingDeviceIndex = index; m_OpenAt = Time.realtimeSinceStartup + m_OpenCooldownSeconds; } void OpenPendingDevice() { var index = m_PendingDeviceIndex; m_PendingDeviceIndex = -1; var devices = WebCamTexture.devices; if (devices.Length == 0) return; index = ((index % devices.Length) + devices.Length) % devices.Length; var name = devices[index].name; m_Probing = true; m_ProbeDeadline = Time.realtimeSinceStartup + m_ProbeTimeoutSeconds; m_WebCam = new WebCamTexture(name, m_RequestedWidth, m_RequestedHeight, m_RequestedFps); try { m_WebCam.Play(); } catch (Exception ex) { Debug.LogWarning("[Vision] Failed to start camera '" + name + "': " + ex.Message); m_WebCam = null; m_Probing = false; // Back off (another cooldown) before the next device so we don't hammer the HAL. m_PendingDeviceIndex = (index + 1) % devices.Length; m_OpenAt = Time.realtimeSinceStartup + m_OpenCooldownSeconds; return; } m_DeviceIndex = index; Debug.Log($"[Vision] Camera started: {name} (index {index + 1}/{devices.Length})"); } int ResolveInitialIndex(WebCamDevice[] devices) { if (!string.IsNullOrWhiteSpace(m_DeviceName)) { for (var i = 0; i < devices.Length; i++) if (string.Equals(devices[i].name, m_DeviceName.Trim(), StringComparison.Ordinal)) return i; Debug.LogWarning("[Vision] Camera '" + m_DeviceName + "' not found; using device 0."); } return 0; } public void StopCamera() { m_PendingDeviceIndex = -1; // cancel any scheduled open if (m_WebCam != null) { if (m_WebCam.isPlaying) m_WebCam.Stop(); m_WebCam = null; } if (m_ReadbackTex != null) { Destroy(m_ReadbackTex); m_ReadbackTex = null; } if (m_PumpRt != null) { m_PumpRt.Release(); Destroy(m_PumpRt); m_PumpRt = null; } } public Texture GetLatestFrameTexture() { return HasFrame ? m_WebCam : null; } public event System.Action FrameJpegCaptured; public byte[] GetLatestFrameJpeg() { if (!HasFrame) return null; if (!m_LoggedOrientation) { m_LoggedOrientation = true; Debug.Log($"[Vision] capture orientation: {m_WebCam.deviceName} {m_WebCam.width}x{m_WebCam.height} angle={m_WebCam.videoRotationAngle} mirrored={m_WebCam.videoVerticallyMirrored}"); } // Bake the sensor orientation into the pixels (Android WebCamTexture frames arrive rotated by // videoRotationAngle and may be vertically mirrored). Without this the JPEG sent to vision and // shown in the UI is sideways, even though the live preview corrects it via its transform. var pixels = ToUpright(m_WebCam.GetPixels32(), m_WebCam.width, m_WebCam.height, ((m_WebCam.videoRotationAngle % 360) + 360) % 360, m_WebCam.videoVerticallyMirrored, out var w, out var h); EnsureReadbackTexture(w, h); m_ReadbackTex.SetPixels32(pixels); m_ReadbackTex.Apply(false); var jpeg = m_ReadbackTex.EncodeToJPG(m_JpegQuality); FrameJpegCaptured?.Invoke(jpeg); return jpeg; } // Vertical mirror (if flagged) then clockwise rotation by `angle`, matching the preview correction. static Color32[] ToUpright(Color32[] src, int w, int h, int angle, bool mirror, out int ow, out int oh) { if (mirror) { var flipped = new Color32[src.Length]; for (var y = 0; y < h; y++) System.Array.Copy(src, y * w, flipped, (h - 1 - y) * w, w); src = flipped; } if (angle == 90 || angle == 270) { ow = h; oh = w; var dst = new Color32[src.Length]; for (var y = 0; y < h; y++) for (var x = 0; x < w; x++) dst[angle == 90 ? (w - 1 - x) * ow + y : x * ow + (h - 1 - y)] = src[y * w + x]; return dst; } ow = w; oh = h; if (angle == 180) { var dst = new Color32[src.Length]; for (var y = 0; y < h; y++) for (var x = 0; x < w; x++) dst[(h - 1 - y) * w + (w - 1 - x)] = src[y * w + x]; return dst; } return src; } void EnsureReadbackTexture(int width, int height) { if (m_ReadbackTex != null && m_ReadbackTex.width == width && m_ReadbackTex.height == height) return; if (m_ReadbackTex != null) Destroy(m_ReadbackTex); m_ReadbackTex = new Texture2D(width, height, TextureFormat.RGB24, false); } } }