Spaces:

josephrw
/

live-code-camera-engine

Paused

File size: 8,310 Bytes
<!doctype html>
<html lang="en">
<head>
  <meta charset="utf-8" />
  <title>Live Code Camera Engine</title>
  <meta name="viewport" content="width=device-width, initial-scale=1" />
  <style>
    :root { font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }
    body { margin: 0; background: #070910; color: #f6f8ff; }
    main { max-width: 1240px; margin: 0 auto; padding: 18px; }
    h1 { margin: 0 0 6px; font-size: 28px; }
    .sub { color: #aeb9ca; margin-bottom: 14px; }
    .grid { display: grid; grid-template-columns: 1.05fr 0.95fr; gap: 16px; align-items: start; }
    @media (max-width: 900px) { .grid { grid-template-columns: 1fr; } }
    video, canvas { width: 100%; background: #111827; border-radius: 16px; border: 1px solid #273142; }
    textarea, select, input, button {
      width: 100%; box-sizing: border-box; border-radius: 12px; border: 1px solid #2f3a4e;
      background: #101827; color: #f5f7fb; padding: 11px; font-size: 15px;
    }
    button { cursor: pointer; background: #1f6feb; border: none; font-weight: 750; margin-top: 8px; }
    button.secondary { background: #2b3445; }
    button.danger { background: #8b1e3f; }
    pre { white-space: pre-wrap; overflow-wrap: anywhere; background: #05070d; border: 1px solid #273142; padding: 14px; border-radius: 14px; min-height: 220px; max-height: 520px; overflow-y: auto; }
    label { display:block; margin: 11px 0 6px; color: #b9c2d0; font-size: 13px; }
    .row { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; }
    .status { color: #9fb1c7; font-size: 14px; margin-top: 8px; }
    .metrics { display:grid; grid-template-columns: repeat(3, 1fr); gap:8px; margin-top:10px; }
    .metric { background:#111827; border:1px solid #273142; border-radius:12px; padding:10px; font-size:13px; color:#b9c2d0;}
    .metric strong { color:#fff; display:block; font-size:18px; }
  </style>
</head>
<body>
<main>
  <h1>Live Code Camera Engine</h1>
  <div class="sub">Camera + audio transcript stream → evidence compression → multimodal LLM → live code updates.</div>

  <div class="grid">
    <section>
      <video id="video" autoplay playsinline muted></video>
      <canvas id="canvas" style="display:none"></canvas>

      <div class="row">
        <button id="startBtn">Start live engine</button>
        <button id="stopBtn" class="danger">Stop</button>
      </div>

      <div class="row">
        <button id="speechBtn" class="secondary">Start speech</button>
        <button id="forceBtn" class="secondary">Force synthesize now</button>
      </div>

      <div class="metrics">
        <div class="metric"><strong id="entropy">0</strong>visual entropy</div>
        <div class="metric"><strong id="delta">0</strong>frame delta</div>
        <div class="metric"><strong id="buffered">0</strong>frames buffered</div>
      </div>

      <p id="status" class="status">Idle.</p>
    </section>

    <section>
      <label>Mode</label>
      <select id="mode">
        <option value="continuous_code">continuous_code</option>
        <option value="debug_visible_error">debug_visible_error</option>
        <option value="derive_pipeline">derive_pipeline</option>
        <option value="soft_qr_xlc">soft_qr_xlc</option>
        <option value="research_prism">research_prism</option>
      </select>

      <label>Instruction</label>
      <textarea id="instruction" rows="4" placeholder="Example: As I point the camera around, infer what app/code/pipeline should be generated."></textarea>

      <label>Live speech transcript</label>
      <textarea id="transcript" rows="5" placeholder="Speech transcript appears here if supported. You can type too."></textarea>

      <label>Frame interval, milliseconds</label>
      <input id="frameMs" value="1200" />

      <label>LLM synthesis interval, seconds</label>
      <input id="synthSec" value="8" />
    </section>
  </div>

  <h2>Live generated code / reasoning summary</h2>
  <pre id="output"></pre>

  <h2>Latest receipt</h2>
  <pre id="receipt"></pre>
</main>

<script>
const video = document.getElementById("video");
const canvas = document.getElementById("canvas");
const statusEl = document.getElementById("status");
const transcriptEl = document.getElementById("transcript");
const outputEl = document.getElementById("output");
const receiptEl = document.getElementById("receipt");

let stream = null;
let ws = null;
let frameTimer = null;
let recognition = null;
let sessionId = null;

function setStatus(msg) { statusEl.textContent = msg; }

function wsUrl() {
  const scheme = location.protocol === "https:" ? "wss" : "ws";
  return `${scheme}://${location.host}/ws/live-code`;
}

async function startCamera() {
  stream = await navigator.mediaDevices.getUserMedia({
    video: { facingMode: "environment", width: { ideal: 1280 }, height: { ideal: 720 } },
    audio: true
  });
  video.srcObject = stream;
}

function connectWs() {
  ws = new WebSocket(wsUrl());
  ws.onopen = () => setStatus("WebSocket connected.");
  ws.onmessage = (event) => {
    const msg = JSON.parse(event.data);
    if (msg.type === "session") {
      sessionId = msg.session_id;
      setStatus("Session: " + sessionId);
    } else if (msg.type === "frame_ack") {
      document.getElementById("entropy").textContent = msg.entropy;
      document.getElementById("delta").textContent = msg.delta;
      document.getElementById("buffered").textContent = msg.buffered;
    } else if (msg.type === "synthesis") {
      outputEl.textContent = msg.output + "\n\n---\n\n" + outputEl.textContent;
      receiptEl.textContent = JSON.stringify(msg.receipt, null, 2);
      setStatus("Live synthesis received.");
    } else if (msg.type === "error") {
      outputEl.textContent = "ERROR: " + msg.message + "\n\n" + outputEl.textContent;
      setStatus("Backend error.");
    }
  };
  ws.onclose = () => setStatus("WebSocket closed.");
}

function captureFrameDataUrl() {
  const w = video.videoWidth || 1280;
  const h = video.videoHeight || 720;
  canvas.width = w;
  canvas.height = h;
  const ctx = canvas.getContext("2d");
  ctx.drawImage(video, 0, 0, w, h);
  return canvas.toDataURL("image/jpeg", 0.74);
}

function sendFrame(force=false) {
  if (!ws || ws.readyState !== WebSocket.OPEN) return;
  const dataUrl = captureFrameDataUrl();
  ws.send(JSON.stringify({
    type: "frame",
    frame: dataUrl,
    transcript: transcriptEl.value,
    instruction: document.getElementById("instruction").value,
    mode: document.getElementById("mode").value,
    auto: true,
    force,
    synthesis_interval: Number(document.getElementById("synthSec").value || 8)
  }));
  if (transcriptEl.value.trim()) {
    ws.send(JSON.stringify({ type: "transcript", text: transcriptEl.value }));
  }
}

async function startLive() {
  await startCamera();
  connectWs();
  const ms = Math.max(300, Number(document.getElementById("frameMs").value || 1200));
  frameTimer = setInterval(() => sendFrame(false), ms);
  setStatus("Live camera stream active.");
}

function stopLive() {
  if (frameTimer) clearInterval(frameTimer);
  frameTimer = null;
  if (recognition) recognition.stop();
  if (stream) {
    stream.getTracks().forEach(t => t.stop());
    stream = null;
  }
  if (ws) ws.close();
  setStatus("Stopped.");
}

function startSpeech() {
  const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
  if (!SpeechRecognition) {
    setStatus("SpeechRecognition unavailable in this browser. Type transcript manually.");
    return;
  }
  recognition = new SpeechRecognition();
  recognition.lang = "en-US";
  recognition.continuous = true;
  recognition.interimResults = true;
  recognition.onresult = (event) => {
    let text = "";
    for (let i = 0; i < event.results.length; i++) {
      text += event.results[i][0].transcript + " ";
    }
    transcriptEl.value = text.trim();
  };
  recognition.onerror = (e) => setStatus("Speech error: " + e.error);
  recognition.start();
  setStatus("Speech recognition active.");
}

document.getElementById("startBtn").onclick = () => startLive().catch(e => setStatus(e.message));
document.getElementById("stopBtn").onclick = stopLive;
document.getElementById("speechBtn").onclick = startSpeech;
document.getElementById("forceBtn").onclick = () => sendFrame(true);
</script>
</body>
</html>