chore: merge master → dev/video-fer (live camera FER mode)

New from master:
- Live camera page with real-time browser-side FER
- use-fer.ts / use-streaming-transcription.ts hooks
- Sidebar /live link, warmup route, next.config.ts updates

Dockerfile downloads emotion_model_web.onnx from GitHub at build
time (HF Xet restriction prevents pushing binary via git).

Dockerfile

@@ -36,6 +36,11 @@ RUN cd web && npm ci
 
 COPY web/ web/
 
+# Download browser-side FER model (ONNX) — not in HF Space git (Xet restriction)
+RUN curl -fsSL \
+    "https://raw.githubusercontent.com/aytoast/ser/master/web/public/emotion_model_web.onnx" \
+    -o web/public/emotion_model_web.onnx
+
 # Build with empty API base → browser uses relative paths → nginx routes /api/*
 RUN cd web && NEXT_PUBLIC_API_URL="" npm run build \
     && cp -r public .next/standalone/public \

web/.gitignore

@@ -39,3 +39,7 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+
+# ONNX Runtime WASM (copied from node_modules at build time)
+public/*.wasm
+public/*.mjs

web/next.config.ts

@@ -2,6 +2,8 @@ import type { NextConfig } from "next";
 
 const nextConfig: NextConfig = {
   output: "standalone",
+  // Allow Turbopack (Next.js 16 default) — WASM files served from public/
+  turbopack: {},
 };
 
 export default nextConfig;

web/package-lock.json

@@ -16,6 +16,7 @@
         "framer-motion": "^12.34.3",
         "lucide-react": "^0.575.0",
         "next": "16.1.6",
+        "onnxruntime-web": "^1.20.1",
         "radix-ui": "^1.4.3",
         "react": "19.2.3",
         "react-dom": "19.2.3",
@@ -1985,6 +1986,70 @@
         "react-dom": ">= 16.8"
       }
     },
+    "node_modules/@protobufjs/aspromise": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
+      "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/base64": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
+      "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/codegen": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
+      "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/eventemitter": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
+      "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/fetch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
+      "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "@protobufjs/aspromise": "^1.1.1",
+        "@protobufjs/inquire": "^1.1.0"
+      }
+    },
+    "node_modules/@protobufjs/float": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
+      "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/inquire": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
+      "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/path": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
+      "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/pool": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
+      "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/utf8": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
+      "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==",
+      "license": "BSD-3-Clause"
+    },
     "node_modules/@radix-ui/number": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.1.tgz",
@@ -4018,7 +4083,6 @@
       "version": "20.19.35",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.35.tgz",
       "integrity": "sha512-Uarfe6J91b9HAUXxjvSOdiO2UPOKLm07Q1oh0JHxoZ1y8HoqxDAu3gVrsrOHeiio0kSsoVBt4wFrKOm0dKxVPQ==",
-      "dev": true,
       "license": "MIT",
       "dependencies": {
         "undici-types": "~6.21.0"
@@ -7046,6 +7110,12 @@
         "node": ">=16"
       }
     },
+    "node_modules/flatbuffers": {
+      "version": "1.12.0",
+      "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz",
+      "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==",
+      "license": "SEE LICENSE IN LICENSE.txt"
+    },
     "node_modules/flatted": {
       "version": "3.3.3",
       "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz",
@@ -7424,6 +7494,12 @@
         "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0"
       }
     },
+    "node_modules/guid-typescript": {
+      "version": "1.0.9",
+      "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz",
+      "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==",
+      "license": "ISC"
+    },
     "node_modules/has-bigints": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.1.0.tgz",
@@ -8850,6 +8926,12 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/long": {
+      "version": "5.3.2",
+      "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz",
+      "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==",
+      "license": "Apache-2.0"
+    },
     "node_modules/loose-envify": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
@@ -9520,6 +9602,26 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/onnxruntime-common": {
+      "version": "1.20.1",
+      "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.20.1.tgz",
+      "integrity": "sha512-YiU0s0IzYYC+gWvqD1HzLc46Du1sXpSiwzKb63PACIJr6LfL27VsXSXQvt68EzD3V0D5Bc0vyJTjmMxp0ylQiw==",
+      "license": "MIT"
+    },
+    "node_modules/onnxruntime-web": {
+      "version": "1.20.1",
+      "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.20.1.tgz",
+      "integrity": "sha512-TePF6XVpLL1rWVMIl5Y9ACBQcyCNFThZON/jgElNd9Txb73CIEGlklhYR3UEr1cp5r0rbGI6nDwwrs79g7WjoA==",
+      "license": "MIT",
+      "dependencies": {
+        "flatbuffers": "^1.12.0",
+        "guid-typescript": "^1.0.9",
+        "long": "^5.2.3",
+        "onnxruntime-common": "1.20.1",
+        "platform": "^1.3.6",
+        "protobufjs": "^7.2.4"
+      }
+    },
     "node_modules/open": {
       "version": "11.0.0",
       "resolved": "https://registry.npmjs.org/open/-/open-11.0.0.tgz",
@@ -9785,6 +9887,12 @@
         "node": ">=16.20.0"
       }
     },
+    "node_modules/platform": {
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz",
+      "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==",
+      "license": "MIT"
+    },
     "node_modules/possible-typed-array-names": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz",
@@ -9913,6 +10021,30 @@
         "react-is": "^16.13.1"
       }
     },
+    "node_modules/protobufjs": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz",
+      "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==",
+      "hasInstallScript": true,
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "@protobufjs/aspromise": "^1.1.2",
+        "@protobufjs/base64": "^1.1.2",
+        "@protobufjs/codegen": "^2.0.4",
+        "@protobufjs/eventemitter": "^1.1.0",
+        "@protobufjs/fetch": "^1.1.0",
+        "@protobufjs/float": "^1.0.2",
+        "@protobufjs/inquire": "^1.1.0",
+        "@protobufjs/path": "^1.1.2",
+        "@protobufjs/pool": "^1.1.0",
+        "@protobufjs/utf8": "^1.1.0",
+        "@types/node": ">=13.7.0",
+        "long": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/proxy-addr": {
       "version": "2.0.7",
       "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
@@ -11669,7 +11801,6 @@
       "version": "6.21.0",
       "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
       "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
-      "dev": true,
       "license": "MIT"
     },
     "node_modules/unicorn-magic": {

web/package.json

@@ -17,6 +17,7 @@
     "framer-motion": "^12.34.3",
     "lucide-react": "^0.575.0",
     "next": "16.1.6",
+    "onnxruntime-web": "^1.20.1",
     "radix-ui": "^1.4.3",
     "react": "19.2.3",
     "react-dom": "19.2.3",

web/src/app/api/warmup/route.ts

@@ -0,0 +1,21 @@
+/**
+ * Warmup endpoint: pings the Modal API /health to wake the GPU container.
+ * Modal containers have a 5-min scaledown window, so hitting /health
+ * during page load ensures the container is warm when recording starts.
+ */
+
+const MODAL_API_URL =
+  process.env.MODAL_API_URL ??
+  "https://yongkang-zou1999--evoxtral-api-evoxtralmodel-web.modal.run";
+
+export async function GET() {
+  try {
+    const res = await fetch(`${MODAL_API_URL}/health`, {
+      signal: AbortSignal.timeout(60_000), // Modal cold start can take up to 60s
+    });
+    const data = await res.json();
+    return Response.json(data);
+  } catch {
+    return Response.json({ status: "unreachable" }, { status: 502 });
+  }
+}

web/src/app/live/page.tsx

@@ -0,0 +1,308 @@
+"use client"
+
+import React, { useCallback, useEffect, useRef, useState } from "react"
+import Link from "next/link"
+import { ArrowLeft, VideoCamera, CircleNotch } from "@phosphor-icons/react"
+import { useFER, FER_LABELS, type EmotionScores } from "@/hooks/use-fer"
+import { useStreamingTranscription } from "@/hooks/use-streaming-transcription"
+
+// ── Emotion bars overlay (bottom-left, transparent) ─────────────────────
+function EmotionBars({ scores }: { scores: EmotionScores }) {
+  return (
+    <div className="absolute bottom-3 left-3 w-56 rounded-xl bg-black/40 backdrop-blur-md px-4 py-3">
+      <div className="space-y-1.5">
+        {FER_LABELS.map((label) => {
+          const value = scores[label] ?? 0
+          return (
+            <div key={label} className="flex items-center gap-2">
+              <div className="flex-1 h-[5px] bg-white/10 rounded-full overflow-hidden">
+                <div
+                  className="h-full rounded-full bg-white/70 transition-all duration-300 ease-out"
+                  style={{ width: `${Math.max(value * 100, 0.5)}%` }}
+                />
+              </div>
+              <span className="text-[10px] text-white/70 font-medium w-14 text-right">
+                {label}
+              </span>
+            </div>
+          )
+        })}
+      </div>
+    </div>
+  )
+}
+
+// ── Warmup steps ────────────────────────────────────────────────────────
+type WarmupStep = { label: string; status: "pending" | "loading" | "done" | "error" }
+
+// ── Main page ───────────────────────────────────────────────────────────
+export default function LivePage() {
+  const videoRef = useRef<HTMLVideoElement>(null)
+  const transcriptRef = useRef<HTMLDivElement>(null)
+  const streamRef = useRef<MediaStream | null>(null)
+  const ferIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null)
+
+  const [cameraReady, setCameraReady] = useState(false)
+  const [permissionError, setPermissionError] = useState<string | null>(null)
+  const [warmupDone, setWarmupDone] = useState(false)
+  const [warmupSteps, setWarmupSteps] = useState<WarmupStep[]>([
+    { label: "Camera & Microphone", status: "pending" },
+    { label: "FER Model (emotion detection)", status: "pending" },
+    { label: "Evoxtral API (transcription)", status: "pending" },
+  ])
+
+  const { isLoaded: ferLoaded, scores, preload: preloadFER, classify } = useFER()
+  const { isRecording, isTranscribing, transcript, currentChunk, start, stop, reset } =
+    useStreamingTranscription()
+
+  // ── Warmup sequence ─────────────────────────────────────────────────
+  useEffect(() => {
+    let cancelled = false
+
+    const updateStep = (idx: number, status: WarmupStep["status"]) => {
+      setWarmupSteps((prev) => prev.map((s, i) => (i === idx ? { ...s, status } : s)))
+    }
+
+    async function warmup() {
+      // Step 1: Camera & Mic
+      updateStep(0, "loading")
+      try {
+        const stream = await navigator.mediaDevices.getUserMedia({
+          video: true,
+          audio: true,
+        })
+        if (cancelled) {
+          stream.getTracks().forEach((t) => t.stop())
+          return
+        }
+        streamRef.current = stream
+        if (videoRef.current) {
+          videoRef.current.srcObject = stream
+        }
+        setCameraReady(true)
+        updateStep(0, "done")
+      } catch (err: unknown) {
+        if (cancelled) return
+        updateStep(0, "error")
+        if (err instanceof DOMException && err.name === "NotAllowedError") {
+          setPermissionError(
+            "Camera and microphone access was denied. Please allow permissions and reload."
+          )
+        } else {
+          setPermissionError(
+            "Could not access camera or microphone. Check your device settings."
+          )
+        }
+        return
+      }
+
+      // Step 2: FER model (parallel with step 3)
+      updateStep(1, "loading")
+      const ferPromise = preloadFER()
+        .then(() => { if (!cancelled) updateStep(1, "done") })
+        .catch(() => { if (!cancelled) updateStep(1, "error") })
+
+      // Step 3: Evoxtral API warmup — actually wake the Modal GPU container
+      updateStep(2, "loading")
+      const apiPromise = fetch("/api/warmup")
+        .then((r) => r.json())
+        .then((data) => {
+          if (!cancelled) updateStep(2, data.status === "ok" ? "done" : "error")
+        })
+        .catch(() => { if (!cancelled) updateStep(2, "error") })
+
+      await Promise.all([ferPromise, apiPromise])
+      if (!cancelled) setWarmupDone(true)
+    }
+
+    warmup()
+
+    return () => {
+      cancelled = true
+      streamRef.current?.getTracks().forEach((t) => t.stop())
+      streamRef.current = null
+    }
+  }, [preloadFER])
+
+  // ── FER classification interval (always on when camera ready) ────────
+  useEffect(() => {
+    if (!ferLoaded || !cameraReady) return
+
+    ferIntervalRef.current = setInterval(() => {
+      if (videoRef.current) {
+        classify(videoRef.current)
+      }
+    }, 500)
+
+    return () => {
+      if (ferIntervalRef.current) clearInterval(ferIntervalRef.current)
+    }
+  }, [ferLoaded, cameraReady, classify])
+
+  // ── Auto-scroll transcript ────────────────────────────────────────────
+  useEffect(() => {
+    if (transcriptRef.current) {
+      transcriptRef.current.scrollTop = transcriptRef.current.scrollHeight
+    }
+  }, [transcript, currentChunk])
+
+  // ── Handlers ──────────────────────────────────────────────────────────
+  const handleToggleRecording = useCallback(() => {
+    if (isRecording) {
+      stop()
+    } else {
+      // Reuse the existing camera+mic stream so we don't request permissions again
+      start(streamRef.current ?? undefined)
+    }
+  }, [isRecording, start, stop])
+
+  const hasScores = Object.keys(scores).length > 0
+
+  // ── Render ────────────────────────────────────────────────────────────
+  return (
+    <div className="flex flex-col min-h-screen bg-background text-foreground">
+      {/* Header */}
+      <header className="flex items-center justify-between px-6 py-3 border-b border-border/40">
+        <Link
+          href="/"
+          className="flex items-center gap-1.5 text-sm text-muted-foreground hover:text-foreground transition-colors"
+        >
+          <ArrowLeft size={16} weight="bold" />
+          <span>Back to Studio</span>
+        </Link>
+
+        <span className="text-sm font-semibold tracking-widest uppercase text-muted-foreground">
+          Live Mode
+        </span>
+
+        <div className="flex items-center gap-2">
+          {isRecording && (
+            <span className="relative flex h-2.5 w-2.5">
+              <span className="animate-ping absolute inline-flex h-full w-full rounded-full bg-red-400 opacity-75" />
+              <span className="relative inline-flex rounded-full h-2.5 w-2.5 bg-red-500" />
+            </span>
+          )}
+        </div>
+      </header>
+
+      {/* Body */}
+      <div className="flex-1 flex flex-col items-center gap-6 px-4 py-8 max-w-4xl mx-auto w-full">
+        {/* Camera preview */}
+        <div className="relative w-full aspect-video rounded-2xl overflow-hidden bg-slate-950 border border-border/30 shadow-2xl">
+          {permissionError ? (
+            <div className="absolute inset-0 flex flex-col items-center justify-center gap-3 p-8 text-center">
+              <VideoCamera size={48} className="text-muted-foreground/40" weight="thin" />
+              <p className="text-sm text-muted-foreground max-w-md">{permissionError}</p>
+            </div>
+          ) : !cameraReady ? (
+            <div className="absolute inset-0 flex flex-col items-center justify-center gap-3">
+              <VideoCamera size={48} className="text-muted-foreground/30 animate-pulse" weight="thin" />
+              <p className="text-xs text-muted-foreground/50">Initializing camera...</p>
+            </div>
+          ) : null}
+
+          <video
+            ref={videoRef}
+            autoPlay
+            muted
+            playsInline
+            className="w-full h-full object-cover"
+            style={{ transform: "scaleX(-1)" }}
+          />
+
+          {/* Emotion bars overlay */}
+          {cameraReady && hasScores && <EmotionBars scores={scores} />}
+
+          {/* Record button — centred at bottom of video */}
+          {cameraReady && (
+            <div className="absolute bottom-4 left-1/2 -translate-x-1/2 flex items-center gap-3">
+              <button
+                onClick={handleToggleRecording}
+                disabled={!warmupDone || isTranscribing}
+                className="group relative flex items-center justify-center"
+                aria-label={isRecording ? "Stop recording" : "Start recording"}
+              >
+                {/* outer ring */}
+                <span
+                  className={`absolute size-16 rounded-full border-[3px] transition-colors ${
+                    isRecording ? "border-white" : isTranscribing ? "border-white/30" : "border-white/60"
+                  }`}
+                />
+                {/* inner shape */}
+                <span
+                  className={`transition-all duration-200 ${
+                    !warmupDone || isTranscribing
+                      ? "size-10 rounded-full bg-white/30 animate-pulse"
+                      : isRecording
+                        ? "size-6 rounded-[4px] bg-red-500"
+                        : "size-12 rounded-full bg-red-500 group-hover:bg-red-400"
+                  }`}
+                />
+              </button>
+
+              {/* Reset button */}
+              {(transcript || currentChunk) && !isRecording && !isTranscribing && (
+                <button
+                  onClick={reset}
+                  className="text-[11px] text-white/60 hover:text-white/90 uppercase tracking-wider transition-colors"
+                >
+                  Reset
+                </button>
+              )}
+            </div>
+          )}
+
+          {/* Transcript overlay — bottom-right, glassmorphism */}
+          {cameraReady && (transcript || currentChunk || isTranscribing) && (
+            <div
+              ref={transcriptRef}
+              className="absolute bottom-3 right-3 w-64 max-h-32 overflow-y-auto rounded-xl bg-black/40 backdrop-blur-md px-4 py-3 text-[12px] leading-relaxed text-white/90"
+            >
+              {transcript}
+              {currentChunk && (
+                <span className="text-white/50">{currentChunk}</span>
+              )}
+              {isTranscribing && !currentChunk && (
+                <span className="text-white/40 italic">Transcribing...</span>
+              )}
+            </div>
+          )}
+        </div>
+
+        {/* Warmup progress (shown before ready) */}
+        {!warmupDone && (
+          <div className="w-full rounded-xl bg-muted/30 border border-border/30 p-5 space-y-3">
+            <p className="text-xs font-semibold uppercase tracking-wider text-muted-foreground/60">
+              Warming up...
+            </p>
+            {warmupSteps.map((step, i) => (
+              <div key={i} className="flex items-center gap-3 text-sm">
+                {step.status === "pending" && (
+                  <div className="size-4 rounded-full border-2 border-muted-foreground/20" />
+                )}
+                {step.status === "loading" && (
+                  <CircleNotch size={16} className="text-blue-500 animate-spin" weight="bold" />
+                )}
+                {step.status === "done" && (
+                  <div className="size-4 rounded-full bg-emerald-500 flex items-center justify-center">
+                    <svg width="10" height="10" viewBox="0 0 10 10" fill="none">
+                      <path d="M2 5L4 7L8 3" stroke="white" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round" />
+                    </svg>
+                  </div>
+                )}
+                {step.status === "error" && (
+                  <div className="size-4 rounded-full bg-red-500 flex items-center justify-center">
+                    <span className="text-[10px] text-white font-bold">!</span>
+                  </div>
+                )}
+                <span className={step.status === "done" ? "text-foreground/70" : step.status === "loading" ? "text-foreground" : "text-muted-foreground/50"}>
+                  {step.label}
+                </span>
+              </div>
+            ))}
+          </div>
+        )}
+      </div>
+    </div>
+  )
+}

web/src/components/sidebar.tsx

@@ -9,6 +9,7 @@ import {
   Heart,
   Plus,
   X,
+  Broadcast,
 } from "@phosphor-icons/react"
 import Image from "next/image"
 import {
@@ -95,6 +96,14 @@ export function AppSidebar() {
                   </Link>
                 </SidebarMenuButton>
               </SidebarMenuItem>
+              <SidebarMenuItem>
+                <SidebarMenuButton asChild isActive={pathname === "/live"}>
+                  <Link href="/live">
+                    <Broadcast />
+                    <span>Live Mode</span>
+                  </Link>
+                </SidebarMenuButton>
+              </SidebarMenuItem>
             </SidebarMenu>
           </SidebarGroupContent>
         </SidebarGroup>

web/src/hooks/use-fer.ts

@@ -0,0 +1,172 @@
+"use client"
+
+import { useState, useRef, useCallback, useEffect } from "react"
+
+export const FER_LABELS = [
+  "Anger",
+  "Contempt",
+  "Disgust",
+  "Fear",
+  "Happy",
+  "Neutral",
+  "Sad",
+  "Surprise",
+] as const
+
+export type EmotionScores = Record<string, number>
+
+const IMAGE_SIZE = 224
+
+const IMAGENET_MEAN = [0.485, 0.456, 0.406]
+const IMAGENET_STD = [0.229, 0.224, 0.225]
+
+function softmax(scores: Float32Array): Float32Array {
+  const max = Math.max(...scores)
+  const exps = new Float32Array(scores.length)
+  let sum = 0
+  for (let i = 0; i < scores.length; i++) {
+    exps[i] = Math.exp(scores[i] - max)
+    sum += exps[i]
+  }
+  for (let i = 0; i < exps.length; i++) {
+    exps[i] /= sum
+  }
+  return exps
+}
+
+export function useFER(): {
+  isLoaded: boolean
+  emotion: string | null
+  confidence: number
+  scores: EmotionScores
+  preload: () => Promise<void>
+  classify: (video: HTMLVideoElement) => Promise<void>
+} {
+  const [isLoaded, setIsLoaded] = useState(false)
+  const [emotion, setEmotion] = useState<string | null>(null)
+  const [confidence, setConfidence] = useState(0)
+  const [scores, setScores] = useState<EmotionScores>({})
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const sessionRef = useRef<any>(null)
+  const canvasRef = useRef<HTMLCanvasElement | null>(null)
+  const loadingRef = useRef(false)
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const ortRef = useRef<any>(null)
+
+  const loadModel = useCallback(async () => {
+    if (sessionRef.current || loadingRef.current) return
+    loadingRef.current = true
+
+    try {
+      const ort = await import("onnxruntime-web")
+      ortRef.current = ort
+
+      // Configure WASM before creating session
+      ort.env.wasm.numThreads = 1
+      ort.env.wasm.wasmPaths = "/"
+
+      // Fetch model as ArrayBuffer
+      const response = await fetch("/emotion_model_web.onnx")
+      const modelBuffer = await response.arrayBuffer()
+
+      const session = await ort.InferenceSession.create(
+        new Uint8Array(modelBuffer),
+        { executionProviders: ["wasm"] }
+      )
+      sessionRef.current = session
+      setIsLoaded(true)
+    } catch (err) {
+      console.error("[useFER] Failed to load ONNX model:", err)
+      loadingRef.current = false
+    }
+  }, [])
+
+  const classify = useCallback(
+    async (video: HTMLVideoElement) => {
+      try {
+        if (!sessionRef.current) {
+          await loadModel()
+        }
+
+        const session = sessionRef.current
+        const ort = ortRef.current
+        if (!session || !ort) return
+
+        if (!canvasRef.current) {
+          canvasRef.current = document.createElement("canvas")
+          canvasRef.current.width = IMAGE_SIZE
+          canvasRef.current.height = IMAGE_SIZE
+        }
+
+        const canvas = canvasRef.current
+        const ctx = canvas.getContext("2d", { willReadFrequently: true })
+        if (!ctx) return
+
+        ctx.drawImage(video, 0, 0, IMAGE_SIZE, IMAGE_SIZE)
+        const imageData = ctx.getImageData(0, 0, IMAGE_SIZE, IMAGE_SIZE)
+        const { data } = imageData
+
+        const floatData = new Float32Array(1 * 3 * IMAGE_SIZE * IMAGE_SIZE)
+        const pixelCount = IMAGE_SIZE * IMAGE_SIZE
+
+        for (let i = 0; i < pixelCount; i++) {
+          const srcIdx = i * 4
+          floatData[i] = (data[srcIdx] / 255 - IMAGENET_MEAN[0]) / IMAGENET_STD[0]
+          floatData[pixelCount + i] =
+            (data[srcIdx + 1] / 255 - IMAGENET_MEAN[1]) / IMAGENET_STD[1]
+          floatData[2 * pixelCount + i] =
+            (data[srcIdx + 2] / 255 - IMAGENET_MEAN[2]) / IMAGENET_STD[2]
+        }
+
+        const inputTensor = new ort.Tensor("float32", floatData, [
+          1,
+          3,
+          IMAGE_SIZE,
+          IMAGE_SIZE,
+        ])
+
+        const inputName = session.inputNames[0]
+        const results = await session.run({ [inputName]: inputTensor })
+        const outputName = session.outputNames[0]
+        const output = results[outputName]
+
+        if (!output) return
+
+        const rawScores = output.data as Float32Array
+        const probs = softmax(rawScores)
+
+        const scoreMap: EmotionScores = {}
+        let maxIdx = 0
+        let maxVal = probs[0]
+        for (let i = 0; i < probs.length; i++) {
+          scoreMap[FER_LABELS[i]] = probs[i]
+          if (probs[i] > maxVal) {
+            maxVal = probs[i]
+            maxIdx = i
+          }
+        }
+
+        setScores(scoreMap)
+        setEmotion(FER_LABELS[maxIdx])
+        setConfidence(maxVal)
+      } catch (err) {
+        console.error("[useFER] Classification error:", err)
+      }
+    },
+    [loadModel]
+  )
+
+  useEffect(() => {
+    return () => {
+      if (sessionRef.current) {
+        sessionRef.current.release().catch((err: unknown) => {
+          console.error("[useFER] Failed to release session:", err)
+        })
+        sessionRef.current = null
+      }
+    }
+  }, [])
+
+  return { isLoaded, emotion, confidence, scores, preload: loadModel, classify }
+}

web/src/hooks/use-streaming-transcription.ts

@@ -0,0 +1,175 @@
+"use client"
+
+import { useState, useRef, useCallback, useEffect } from "react"
+
+interface ChunkStreamEvent {
+  token?: string
+  done?: boolean
+  transcription?: string
+}
+
+/**
+ * Press-to-record transcription hook.
+ * 1. start() → begins recording audio
+ * 2. stop()  → stops recording, sends full audio to API, streams back tokens
+ */
+export function useStreamingTranscription() {
+  const [isRecording, setIsRecording] = useState(false)
+  const [isTranscribing, setIsTranscribing] = useState(false)
+  const [transcript, setTranscript] = useState("")
+  const [currentChunk, setCurrentChunk] = useState("")
+
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null)
+  const ownStreamRef = useRef<MediaStream | null>(null)
+  const blobsRef = useRef<Blob[]>([])
+
+  const transcribe = useCallback(async (audioBlob: Blob) => {
+    if (audioBlob.size < 500) return
+
+    setIsTranscribing(true)
+    setCurrentChunk("")
+
+    try {
+      const formData = new FormData()
+      formData.append("audio", audioBlob, "recording.webm")
+
+      const response = await fetch("/api/transcribe-stream", {
+        method: "POST",
+        body: formData,
+      })
+
+      if (!response.ok || !response.body) {
+        console.error("Transcription request failed:", response.status)
+        return
+      }
+
+      const reader = response.body.getReader()
+      const decoder = new TextDecoder()
+      let buffer = ""
+      let streamingText = ""
+
+      while (true) {
+        const { done, value } = await reader.read()
+        if (done) break
+
+        buffer += decoder.decode(value, { stream: true })
+        const lines = buffer.split("\n")
+        buffer = lines.pop() ?? ""
+
+        for (const line of lines) {
+          const trimmed = line.trim()
+          if (!trimmed.startsWith("data: ")) continue
+          const jsonStr = trimmed.slice(6)
+          if (!jsonStr) continue
+
+          try {
+            const event: ChunkStreamEvent = JSON.parse(jsonStr)
+            if (event.done && event.transcription != null) {
+              // Final result — set as transcript
+              setTranscript((prev) =>
+                prev ? prev + " " + event.transcription! : event.transcription!
+              )
+              setCurrentChunk("")
+            } else if (event.token != null) {
+              streamingText += event.token
+              setCurrentChunk(streamingText)
+            }
+          } catch {
+            // ignore malformed JSON
+          }
+        }
+      }
+    } catch (error) {
+      console.error("Transcription error:", error)
+    } finally {
+      setIsTranscribing(false)
+    }
+  }, [])
+
+  const start = useCallback(
+    async (existingStream?: MediaStream) => {
+      if (isRecording) return
+
+      blobsRef.current = []
+
+      let stream: MediaStream
+      if (existingStream) {
+        const audioTracks = existingStream.getAudioTracks()
+        if (audioTracks.length === 0) {
+          console.error("[useStreamingTranscription] No audio tracks")
+          return
+        }
+        stream = new MediaStream(audioTracks)
+      } else {
+        stream = await navigator.mediaDevices.getUserMedia({ audio: true })
+        ownStreamRef.current = stream
+      }
+
+      const mimeType = MediaRecorder.isTypeSupported("audio/webm;codecs=opus")
+        ? "audio/webm;codecs=opus"
+        : "audio/webm"
+
+      const recorder = new MediaRecorder(stream, { mimeType })
+      mediaRecorderRef.current = recorder
+
+      recorder.ondataavailable = (event: BlobEvent) => {
+        if (event.data.size > 0) {
+          blobsRef.current.push(event.data)
+        }
+      }
+
+      recorder.start()
+      setIsRecording(true)
+    },
+    [isRecording]
+  )
+
+  const stop = useCallback(() => {
+    const recorder = mediaRecorderRef.current
+    if (!recorder || recorder.state === "inactive") return
+
+    // When the recorder stops, assemble blobs and send for transcription
+    recorder.onstop = () => {
+      const mimeType = recorder.mimeType || "audio/webm;codecs=opus"
+      const audioBlob = new Blob(blobsRef.current, { type: mimeType })
+      blobsRef.current = []
+      transcribe(audioBlob)
+    }
+
+    recorder.stop()
+    mediaRecorderRef.current = null
+
+    // Only stop tracks if we own the stream
+    if (ownStreamRef.current) {
+      ownStreamRef.current.getTracks().forEach((t) => t.stop())
+      ownStreamRef.current = null
+    }
+
+    setIsRecording(false)
+  }, [transcribe])
+
+  const reset = useCallback(() => {
+    setTranscript("")
+    setCurrentChunk("")
+  }, [])
+
+  useEffect(() => {
+    return () => {
+      const recorder = mediaRecorderRef.current
+      if (recorder && recorder.state !== "inactive") recorder.stop()
+      if (ownStreamRef.current) {
+        ownStreamRef.current.getTracks().forEach((t) => t.stop())
+      }
+    }
+  }, [])
+
+  return {
+    isRecording,
+    isTranscribing,
+    transcript,
+    currentChunk,
+    start,
+    stop,
+    reset,
+  }
+}