web/src/voice/camera.ts

import type { Lang } from "../i18n";

/** Still-image capture from the robot's camera + vision identification.
 *
 *  The robot streams its camera over WebRTC; we read the live video track
 *  from the peer connection's receivers (same pattern as the robot mic) and
 *  grab ONE frame via an off-screen <video> sink — nothing is displayed,
 *  streamed, or recorded beyond that single JPEG.
 *
 *  gpt-realtime doesn't take image input, so the frame goes to a vision model
 *  (gpt-4o-mini, same OpenAI key) and the result is returned to the Realtime
 *  session as the take_picture tool output — the two-step architecture used
 *  by the Python app's camera tool and recommended for Realtime. */

export function getRobotVideoTrack(reachy?: { _pc?: RTCPeerConnection | null } | null): MediaStreamTrack | null {
  const pc = reachy?._pc ?? null;
  if (!pc) return null;
  for (const r of pc.getReceivers()) {
    if (r.track && r.track.kind === "video" && r.track.readyState === "live") return r.track;
  }
  return null;
}

export function countVideoReceivers(reachy?: { _pc?: RTCPeerConnection | null } | null): number {
  const pc = reachy?._pc ?? null;
  if (!pc) return 0;
  return pc.getReceivers().filter((r) => r.track?.kind === "video").length;
}

/** Grab a single still frame from the track as a JPEG data URL (~768px wide). */
export async function captureFrame(track: MediaStreamTrack): Promise<string> {
  const video = document.createElement("video");
  video.muted = true;
  video.autoplay = true;
  video.setAttribute("playsinline", "");
  video.style.cssText = "position:fixed;left:-9999px;top:0;width:2px;height:2px;";
  video.srcObject = new MediaStream([track]);
  document.body.appendChild(video);
  try {
    await video.play().catch(() => {});
    // Wait for real pixels (remote tracks need a moment after attach).
    await new Promise<void>((resolve, reject) => {
      const deadline = setTimeout(() => reject(new Error("camera frame timeout")), 5000);
      const check = () => {
        if (video.videoWidth > 0 && video.readyState >= 2) {
          clearTimeout(deadline);
          // Let the camera's auto-exposure settle — the very first decoded
          // frames come out dark/black.
          setTimeout(resolve, 450);
        } else {
          setTimeout(check, 100);
        }
      };
      check();
    });
    const scale = Math.min(1, 768 / video.videoWidth);
    const canvas = document.createElement("canvas");
    canvas.width = Math.round(video.videoWidth * scale);
    canvas.height = Math.round(video.videoHeight * scale);
    const ctx2d = canvas.getContext("2d");
    if (!ctx2d) throw new Error("no 2d context");
    ctx2d.drawImage(video, 0, 0, canvas.width, canvas.height);
    const dataUrl = canvas.toDataURL("image/jpeg", 0.8);
    console.log(`[voice] captured frame ${canvas.width}x${canvas.height} (${Math.round(dataUrl.length / 1024)} KB)`);
    return dataUrl;
  } finally {
    video.srcObject = null;
    video.remove();
  }
}

export interface VisionResult {
  description: string;
  items: { name: string; quantity?: number; unit?: string }[];
}

/** Ask gpt-4o-mini what's in the photo, kitchen-focused, structured output. */
export async function identifyWithVision(
  apiKey: string,
  dataUrl: string,
  question: string,
  lang: Lang,
): Promise<VisionResult> {
  const langLine = lang === "es" ? "Answer the description in Spanish." : "Answer the description in English.";
  const started = performance.now();
  const res = await fetch("https://api.openai.com/v1/chat/completions", {
    method: "POST",
    headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
    body: JSON.stringify({
      model: "gpt-4o-mini",
      max_tokens: 500,
      response_format: { type: "json_object" },
      messages: [
        {
          role: "system",
          content:
            "You are the eyes of a kitchen-assistant robot. Look at the photo from the robot's camera and answer the user's question about it. " +
            "Also extract any FOOD items visible as inventory candidates (name, estimated quantity and unit among g/kg/ml/l/pcs when guessable). " +
            `Reply ONLY with JSON: {"description": string, "items": [{"name": string, "quantity"?: number, "unit"?: string}]}. ${langLine} ` +
            "Only mention what is actually visible; if the photo is dark or unclear, say so in the description and return an empty items list.",
        },
        {
          role: "user",
          content: [
            { type: "text", text: question || "What do you see?" },
            { type: "image_url", image_url: { url: dataUrl } },
          ],
        },
      ],
    }),
  });
  if (!res.ok) {
    const body = await res.text().catch(() => "");
    throw new Error(`vision request failed (${res.status}): ${body.slice(0, 200)}`);
  }
  const json = (await res.json()) as { choices?: { message?: { content?: string } }[] };
  const content = json.choices?.[0]?.message?.content ?? "";
  console.log(`[voice] vision reply in ${Math.round(performance.now() - started)} ms`);
  try {
    const parsed = JSON.parse(content) as Partial<VisionResult>;
    return {
      description: typeof parsed.description === "string" ? parsed.description : content,
      items: Array.isArray(parsed.items) ? parsed.items.filter((i) => i && typeof i.name === "string") : [],
    };
  } catch {
    return { description: content, items: [] };
  }
}