Runtime/AgentCore/Tools/Builtin/AnalyzeCurrentViewTool.cs

using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using OnDeviceAgent.Inference;
namespace OnDeviceAgent.AgentCore
{

    public sealed class AnalyzeCurrentViewTool : AgentToolBase
    {
        readonly CameraInputService m_Camera;
        readonly Func<string, byte[], CancellationToken, Task<string>> m_VisionAsk;

        public AnalyzeCurrentViewTool(CameraInputService camera, Func<string, byte[], CancellationToken, Task<string>> visionAsk)
        {
            m_Camera = camera;
            m_VisionAsk = visionAsk;
        }

        public override string Name => "AnalyzeCurrentView";
        public override string Description =>
            "Look at the live camera and answer what is currently visible. Use this whenever the user asks to describe, see, or look at what is in front of them or on screen (e.g. 'what do you see', 'describe this view', '눈앞에 보이는 거 설명해줘'). Pass the user's question as 'question'.";

        public override AITool ToAIFunction(AgentToolContext context)
        {
            Task<string> Run([System.ComponentModel.Description("the user's question about the current camera view")] string question)
            {
                var argsJson = "{\"question\":\"" + (question ?? string.Empty).Replace("\"", "'") + "\"}";
                Func<Task<string>> body = null;
                if (m_Camera != null && m_VisionAsk != null)
                    body = async () =>
                    {
                        var jpeg = await context.Dispatcher.RunOnMainAsync<byte[]>(() =>
                            Task.FromResult(m_Camera.HasFrame ? m_Camera.GetLatestFrameJpeg() : null)).ConfigureAwait(false);
                        if (jpeg == null || jpeg.Length == 0)
                            return "The camera has no frame yet.";
                        var q = string.IsNullOrWhiteSpace(question) ? "Describe what you see." : question.Trim();
                        var answer = await m_VisionAsk(q, jpeg, context.CancellationToken).ConfigureAwait(false);
                        return string.IsNullOrWhiteSpace(answer) ? "I couldn't interpret the current view." : answer.Trim();
                    };
                return RunGuarded(context, argsJson, body, "Vision is not configured.");
            }

            return AIFunctionFactory.Create((Func<string, Task<string>>)Run, name: Name, description: Description);
        }
    }
}