Runtime/Inference/Camera/CameraInputService.cs

using System;
using UnityEngine;
namespace OnDeviceAgent.Inference
{

    public sealed class CameraInputService : MonoBehaviour
    {
        [SerializeField] bool m_StartOnEnable = true;
        [SerializeField, Tooltip("Camera device name; empty = system default.")]
        string m_DeviceName = "";
        [SerializeField] int m_RequestedWidth = 1280;
        [SerializeField] int m_RequestedHeight = 720;
        [SerializeField] int m_RequestedFps = 30;
        [SerializeField, Range(10, 100)] int m_JpegQuality = 75;
        [SerializeField, Tooltip("Auto-advance to the next camera if the current one delivers no frames within this many seconds. Headless devices (no preview to tap) need this when device 0 is a non-streaming sensor.")]
        float m_ProbeTimeoutSeconds = 3f;
        [SerializeField, Tooltip("If no camera is streaming (e.g. the HAL was wedged by a previous run), re-probe every this many seconds so it recovers without an app restart.")]
        float m_CameraRetrySeconds = 8f;
        [SerializeField, Tooltip("Wait this long after releasing a camera before opening the next one, so the HAL frees the previous client first.")]
        float m_OpenCooldownSeconds = 0.7f;

        WebCamTexture m_WebCam;
        Texture2D m_ReadbackTex;
        int m_DeviceIndex = -1;
        int m_PendingDeviceIndex = -1;
        float m_OpenAt;

        bool m_Probing;
        int m_ProbeAttempts;
        float m_ProbeDeadline;
        float m_NextProbeLog;
        float m_LastUpdateTime;
        float m_NextRetryAt;
        RenderTexture m_PumpRt;
        bool m_LoggedOrientation;

        public bool IsRunning => m_WebCam != null && m_WebCam.isPlaying;

        public bool HasFrame => IsRunning && m_WebCam.width > 16;

        public string CurrentDeviceName => m_WebCam != null ? m_WebCam.deviceName : null;

        void OnEnable()
        {
            if (m_StartOnEnable)
                StartCamera();
        }

        void OnDisable()
        {
            StopCamera();
        }

        // Headless probe: WebCamTexture.Play() succeeds even on sensors that never deliver frames (width stays
        // at 16). With no preview to tap, auto-advance through the devices until one actually streams.
        void Update()
        {
            var now = Time.realtimeSinceStartup;
            var dt = m_LastUpdateTime > 0f ? now - m_LastUpdateTime : 0f;
            m_LastUpdateTime = now;

            // A device switch is pending: we stopped the previous camera and wait out a cooldown so the HAL
            // releases that client before opening the next. Opening too soon piles up clients until the service
            // rejects every open ("Too many other clients connecting") and no device ever streams.
            if (m_PendingDeviceIndex >= 0)
            {
                if (now >= m_OpenAt)
                    OpenPendingDevice();
                return;
            }

            // Android only pumps the camera SurfaceTexture (UpdateTexImage) when the texture is sampled. With the
            // preview removed nothing renders it, so frames never arrive. Blit it off-screen every frame to force
            // the update even when there is no visible consumer.
            if (m_WebCam != null && m_WebCam.isPlaying)
            {
                if (m_PumpRt == null)
                    m_PumpRt = new RenderTexture(64, 64, 0);
                Graphics.Blit(m_WebCam, m_PumpRt);
            }

            if (!m_Probing)
            {
                MaybeRetryCamera(now);
                return;
            }

            if (m_WebCam != null && m_WebCam.isPlaying && m_WebCam.width > 16)
            {
                m_Probing = false;
                m_NextRetryAt = 0f;
                Debug.Log($"[Vision] Camera streaming: {m_WebCam.deviceName} ({m_WebCam.width}x{m_WebCam.height})");
                return;
            }

            // A long stall (e.g. the ~10s on-device model load blocks the main thread) must not count against the
            // probe window: the camera can't pump while Update isn't ticking, so push the deadline past the stall.
            // Otherwise the deadline expires during the freeze and we thrash every device with rapid Stop/Play,
            // which wedges the camera HAL so none recover.
            if (dt > 0.75f)
                m_ProbeDeadline += dt;

            if (now >= m_NextProbeLog)
            {
                m_NextProbeLog = now + 1f;
                Debug.Log($"[Vision] probe {CurrentDeviceName}: {(m_WebCam != null ? m_WebCam.width : -1)}x{(m_WebCam != null ? m_WebCam.height : -1)} isPlaying={(m_WebCam != null && m_WebCam.isPlaying)} didUpdate={(m_WebCam != null && m_WebCam.didUpdateThisFrame)}");
            }

            if (now < m_ProbeDeadline)
                return;

            var devices = WebCamTexture.devices;
            m_ProbeAttempts++;
            if (m_ProbeAttempts >= devices.Length)
            {
                // Tried every device once and none streamed. Stop cycling and let MaybeRetryCamera re-probe on a
                // slow cadence (the HAL is often just wedged and recovers) instead of thrashing Stop/Play.
                Debug.LogWarning($"[Vision] No camera delivered frames after trying {devices.Length} device(s); re-probing in {m_CameraRetrySeconds}s.");
                m_Probing = false;
                m_NextRetryAt = 0f; // arm MaybeRetryCamera
                return;
            }

            Debug.Log($"[Vision] Camera '{CurrentDeviceName}' delivered no frames in {m_ProbeTimeoutSeconds}s; trying next device.");
            StartDeviceAt((m_DeviceIndex + 1) % devices.Length);
        }

        // After the probe gives up, the camera may still come back (the HAL is often just wedged by a previous
        // run). Re-probe on a slow cadence so it recovers without an app restart, instead of staying dark forever.
        void MaybeRetryCamera(float now)
        {
            if (HasFrame)
            {
                m_NextRetryAt = 0f;
                return;
            }
            if (m_NextRetryAt <= 0f)
            {
                m_NextRetryAt = now + m_CameraRetrySeconds; // arm the wait
                return;
            }
            if (now < m_NextRetryAt)
                return;

            m_NextRetryAt = 0f; // consumed; re-armed once this re-probe gives up again
            Debug.Log("[Vision] Camera not streaming; re-probing devices.");
            StartCamera();
        }

        public void StartCamera()
        {
            var devices = WebCamTexture.devices;
            if (devices.Length == 0)
            {
                Debug.LogWarning("[Vision] No camera device found.");
                return;
            }

            if (m_DeviceIndex < 0 || m_DeviceIndex >= devices.Length)
                m_DeviceIndex = ResolveInitialIndex(devices);

            m_ProbeAttempts = 0;
            StartDeviceAt(m_DeviceIndex);
        }

        // Releases the current camera and schedules the open of `index` after the cooldown; the actual open
        // happens in OpenPendingDevice (driven from Update) so we never hold two camera clients at once.
        void StartDeviceAt(int index)
        {
            var devices = WebCamTexture.devices;
            if (devices.Length == 0)
                return;
            index = ((index % devices.Length) + devices.Length) % devices.Length;

            StopCamera();

            m_Probing = false; // not probing until the device actually opens
            m_PendingDeviceIndex = index;
            m_OpenAt = Time.realtimeSinceStartup + m_OpenCooldownSeconds;
        }

        void OpenPendingDevice()
        {
            var index = m_PendingDeviceIndex;
            m_PendingDeviceIndex = -1;

            var devices = WebCamTexture.devices;
            if (devices.Length == 0)
                return;
            index = ((index % devices.Length) + devices.Length) % devices.Length;

            var name = devices[index].name;
            m_Probing = true;
            m_ProbeDeadline = Time.realtimeSinceStartup + m_ProbeTimeoutSeconds;
            m_WebCam = new WebCamTexture(name, m_RequestedWidth, m_RequestedHeight, m_RequestedFps);
            try
            {
                m_WebCam.Play();
            }
            catch (Exception ex)
            {
                Debug.LogWarning("[Vision] Failed to start camera '" + name + "': " + ex.Message);
                m_WebCam = null;
                m_Probing = false;
                // Back off (another cooldown) before the next device so we don't hammer the HAL.
                m_PendingDeviceIndex = (index + 1) % devices.Length;
                m_OpenAt = Time.realtimeSinceStartup + m_OpenCooldownSeconds;
                return;
            }

            m_DeviceIndex = index;
            Debug.Log($"[Vision] Camera started: {name} (index {index + 1}/{devices.Length})");
        }

        int ResolveInitialIndex(WebCamDevice[] devices)
        {
            if (!string.IsNullOrWhiteSpace(m_DeviceName))
            {
                for (var i = 0; i < devices.Length; i++)
                    if (string.Equals(devices[i].name, m_DeviceName.Trim(), StringComparison.Ordinal))
                        return i;
                Debug.LogWarning("[Vision] Camera '" + m_DeviceName + "' not found; using device 0.");
            }
            return 0;
        }

        public void StopCamera()
        {
            m_PendingDeviceIndex = -1; // cancel any scheduled open
            if (m_WebCam != null)
            {
                if (m_WebCam.isPlaying)
                    m_WebCam.Stop();
                m_WebCam = null;
            }
            if (m_ReadbackTex != null)
            {
                Destroy(m_ReadbackTex);
                m_ReadbackTex = null;
            }
            if (m_PumpRt != null)
            {
                m_PumpRt.Release();
                Destroy(m_PumpRt);
                m_PumpRt = null;
            }
        }

        public Texture GetLatestFrameTexture()
        {
            return HasFrame ? m_WebCam : null;
        }

        public event System.Action<byte[]> FrameJpegCaptured;

        public byte[] GetLatestFrameJpeg()
        {
            if (!HasFrame)
                return null;

            if (!m_LoggedOrientation)
            {
                m_LoggedOrientation = true;
                Debug.Log($"[Vision] capture orientation: {m_WebCam.deviceName} {m_WebCam.width}x{m_WebCam.height} angle={m_WebCam.videoRotationAngle} mirrored={m_WebCam.videoVerticallyMirrored}");
            }

            // Bake the sensor orientation into the pixels (Android WebCamTexture frames arrive rotated by
            // videoRotationAngle and may be vertically mirrored). Without this the JPEG sent to vision and
            // shown in the UI is sideways, even though the live preview corrects it via its transform.
            var pixels = ToUpright(m_WebCam.GetPixels32(), m_WebCam.width, m_WebCam.height,
                ((m_WebCam.videoRotationAngle % 360) + 360) % 360, m_WebCam.videoVerticallyMirrored,
                out var w, out var h);

            EnsureReadbackTexture(w, h);
            m_ReadbackTex.SetPixels32(pixels);
            m_ReadbackTex.Apply(false);
            var jpeg = m_ReadbackTex.EncodeToJPG(m_JpegQuality);
            FrameJpegCaptured?.Invoke(jpeg);
            return jpeg;
        }

        // Vertical mirror (if flagged) then clockwise rotation by `angle`, matching the preview correction.
        static Color32[] ToUpright(Color32[] src, int w, int h, int angle, bool mirror, out int ow, out int oh)
        {
            if (mirror)
            {
                var flipped = new Color32[src.Length];
                for (var y = 0; y < h; y++)
                    System.Array.Copy(src, y * w, flipped, (h - 1 - y) * w, w);
                src = flipped;
            }

            if (angle == 90 || angle == 270)
            {
                ow = h; oh = w;
                var dst = new Color32[src.Length];
                for (var y = 0; y < h; y++)
                    for (var x = 0; x < w; x++)
                        dst[angle == 90 ? (w - 1 - x) * ow + y : x * ow + (h - 1 - y)] = src[y * w + x];
                return dst;
            }

            ow = w; oh = h;
            if (angle == 180)
            {
                var dst = new Color32[src.Length];
                for (var y = 0; y < h; y++)
                    for (var x = 0; x < w; x++)
                        dst[(h - 1 - y) * w + (w - 1 - x)] = src[y * w + x];
                return dst;
            }
            return src;
        }

        void EnsureReadbackTexture(int width, int height)
        {
            if (m_ReadbackTex != null && m_ReadbackTex.width == width && m_ReadbackTex.height == height)
                return;
            if (m_ReadbackTex != null)
                Destroy(m_ReadbackTex);
            m_ReadbackTex = new Texture2D(width, height, TextureFormat.RGB24, false);
        }

    }
}