// ───────────────────────────────────────────────────────────────────────────── // Browser-side screen-capture for the "explainer zoom" feature. // // Records the screen (preferring THIS tab) via getDisplayMedia while sampling the // cursor track + click events off the live DOM, then assembles a ScreencastData // bundle (cursor / clicks / cam) that rides alongside the recorded webm so the // renderer can drive the auto-zoom camera. See screencast.ts for the data model // and screencastZoom.ts for the camera engine. // // Pure DOM / Media APIs — no React. Capture-time only, so the non-determinism // (performance.now / getBoundingClientRect) is fine; the renderer is the part that // must stay frame-deterministic. // ───────────────────────────────────────────────────────────────────────────── import type { ScreencastData, ScreencastPoint, ScreencastClick, ScreencastRect, } from "./screencast"; import { withAutoZoom, type AutoZoomOpts, type ZoomStyle } from "./screencastZoom"; const clamp01 = (v: number) => Math.min(1, Math.max(0, v)); export interface CaptureResult { blob: Blob; durationMs: number; width: number; height: number; screencast: ScreencastData; } export interface CaptureController { stop: () => Promise; cancel: () => void; stream: MediaStream; } // Capture quality presets. getDisplayMedia can't exceed the actual surface's pixel // count, but requesting a high `ideal` makes Chrome capture at the surface's full // (DPR-scaled) resolution. Recording ABOVE the composition size is what keeps the // footage crisp when the explainer camera zooms in (a 2× zoom on 4K source still // has full-HD detail; a 2× zoom on 1080 source is soft). export type CaptureQuality = "1080p" | "1440p" | "4k"; const QUALITY: Record = { "1080p": { w: 1920, h: 1080, bitrate: 8_000_000 }, "1440p": { w: 2560, h: 1440, bitrate: 18_000_000 }, "4k": { w: 3840, h: 2160, bitrate: 44_000_000 }, }; export interface ScreenCaptureOpts { frameRate?: number; // requested capture frame rate (default 30; 60 for buttery motion) quality?: CaptureQuality; // resolution + bitrate preset (default "1440p") animatedCursor?: boolean; // suppress the OS cursor at capture + render our own sprite (default true) cameraStyle?: ZoomStyle; // how eagerly the explainer camera zooms (default "calm") autoZoom?: AutoZoomOpts; // fine-grain tuning passed through to withAutoZoom (overrides cameraStyle) } // Map getDisplayMedia's displaySurface to our coarse ScreencastData.source bucket. function mapSource(surface?: string): ScreencastData["source"] { if (surface === "browser") return "tab"; if (surface === "window") return "window"; return "screen"; // "monitor" / unknown } // Walk up from a hit-tested element to the nearest "meaningful" target (a thing a // user would think they clicked) — so the framing rect is the button/link, not a // stray text node. Capped to a few levels so we never balloon to a layout wrapper. function meaningfulEl(start: Element | null): Element | null { let el: Element | null = start; for (let i = 0; el && i < 4; i++) { const tag = el.tagName.toLowerCase(); if ( el.getAttribute("aria-label") || el.getAttribute("role") === "button" || tag === "button" || tag === "a" || tag === "input" || el.getAttribute("data-testid") ) return el; el = el.parentElement; } return start; } function elLabel(el: Element): string { return ( el.getAttribute("aria-label") || el.getAttribute("title") || (el.textContent || "").trim().slice(0, 40) || el.tagName.toLowerCase() ); } // ── cursor smoothing: a 1€ filter ──────────────────────────────────────────── // EMA lags behind fast motion (the cursor visibly trails the real pointer). The 1€ // filter (Casiez et al.) raises its cutoff with speed: heavy smoothing when the // cursor is slow/still (kills jitter), almost no smoothing when it's moving fast // (no lag) — exactly the behavior a cursor wants. Then we downsample to ~targetHz. const lowpass = (a: number, x: number, prev: number) => a * x + (1 - a) * prev; const alphaFor = (cutoff: number, dt: number) => { const tau = 1 / (2 * Math.PI * cutoff); return 1 / (1 + tau / dt); }; function smoothCursor(raw: ScreencastPoint[], targetHz = 30): ScreencastPoint[] { if (raw.length === 0) return raw; const sorted = [...raw].sort((a, b) => a.t - b.t); const minCutoff = 1.6, beta = 0.45, dCutoff = 1.0; // tuned for 0..1 coordinate space let xP = sorted[0].x, yP = sorted[0].y, dxP = 0, dyP = 0, tP = sorted[0].t; const filtered: ScreencastPoint[] = [{ t: Math.round(tP), x: clamp01(xP), y: clamp01(yP) }]; for (let i = 1; i < sorted.length; i++) { const p = sorted[i]; const dt = Math.max(1, p.t - tP) / 1000; // seconds const dx = (p.x - xP) / dt, dy = (p.y - yP) / dt; const ad = alphaFor(dCutoff, dt); dxP = lowpass(ad, dx, dxP); dyP = lowpass(ad, dy, dyP); const ax = alphaFor(minCutoff + beta * Math.abs(dxP), dt); const ay = alphaFor(minCutoff + beta * Math.abs(dyP), dt); xP = lowpass(ax, p.x, xP); yP = lowpass(ay, p.y, yP); tP = p.t; filtered.push({ t: Math.round(p.t), x: clamp01(xP), y: clamp01(yP) }); } // downsample to ~targetHz, always keeping the first & last const minGap = 1000 / targetHz; const out: ScreencastPoint[] = [filtered[0]]; for (const p of filtered) if (p.t - out[out.length - 1].t >= minGap) out.push(p); const last = filtered[filtered.length - 1]; if (out[out.length - 1].t < last.t) out.push(last); return out; } export async function startScreenCapture(opts: ScreenCaptureOpts = {}): Promise { const fps = opts.frameRate ?? 30; const q = QUALITY[opts.quality ?? "1440p"]; const animatedCursor = opts.animatedCursor !== false; // default true const stream = await navigator.mediaDevices.getDisplayMedia({ video: { frameRate: { ideal: fps, max: fps }, width: { ideal: q.w }, height: { ideal: q.h }, // suppress the OS cursor in the recorded pixels so it doesn't double up with our // animated sprite (best-effort: some browsers ignore it — hence the render-side flag) cursor: animatedCursor ? "never" : "always", }, audio: false, // captures THIS tab when the user agrees → DOM rects map to captured pixels preferCurrentTab: true, } as DisplayMediaStreamOptions); const track = stream.getVideoTracks()[0]; const settings = track?.getSettings?.() ?? {}; const source = mapSource((settings as MediaTrackSettings & { displaySurface?: string }).displaySurface); const isTab = source === "tab"; // ── metadata sampling ────────────────────────────────────────────────────── const t0 = performance.now(); const cursor: ScreencastPoint[] = []; const clicks: ScreencastClick[] = []; const norm = (e: { clientX: number; clientY: number }) => ({ x: clamp01(e.clientX / window.innerWidth), y: clamp01(e.clientY / window.innerHeight), }); let lastMove = 0; const onMove = (e: PointerEvent) => { const now = performance.now(); if (now - lastMove < 33) return; // ~30Hz throttle (smoothed/downsampled later) lastMove = now; const { x, y } = norm(e); cursor.push({ t: now - t0, x, y }); }; const recordClick = (e: PointerEvent | MouseEvent, kind: ScreencastClick["kind"]) => { const { x, y } = norm(e); const click: ScreencastClick = { t: performance.now() - t0, x, y, kind }; // Only attach element geometry when capturing this tab — otherwise the captured // pixels are a different surface and DOM coords wouldn't map reliably. if (isTab) { const hit = document.elementFromPoint(e.clientX, e.clientY); const el = meaningfulEl(hit); if (el) { const r = el.getBoundingClientRect(); const W = window.innerWidth, H = window.innerHeight; const rect: ScreencastRect = { x: clamp01(r.left / W), y: clamp01(r.top / H), w: clamp01(r.width / W), h: clamp01(r.height / H), }; click.rect = rect; click.label = elLabel(el); } } clicks.push(click); }; const onDown = (e: PointerEvent) => recordClick(e, "click"); const onDbl = (e: MouseEvent) => recordClick(e, "dblclick"); const onCtx = (e: MouseEvent) => recordClick(e, "contextmenu"); // capture phase + passive so we observe without interfering with the page const listenOpts: AddEventListenerOptions = { capture: true, passive: true }; window.addEventListener("pointermove", onMove, listenOpts); window.addEventListener("pointerdown", onDown, listenOpts); window.addEventListener("dblclick", onDbl, listenOpts); window.addEventListener("contextmenu", onCtx, listenOpts); const removeListeners = () => { window.removeEventListener("pointermove", onMove, listenOpts); window.removeEventListener("pointerdown", onDown, listenOpts); window.removeEventListener("dblclick", onDbl, listenOpts); window.removeEventListener("contextmenu", onCtx, listenOpts); }; // ── recorder ──────────────────────────────────────────────────────────────── const mime = ["video/webm;codecs=vp9", "video/webm;codecs=vp8", "video/webm"] .find((m) => MediaRecorder.isTypeSupported?.(m)) || ""; // scale the target bitrate with frame rate so 60fps stays as crisp as 30fps const videoBitsPerSecond = Math.round(q.bitrate * (fps / 30)); const rec = new MediaRecorder(stream, { ...(mime ? { mimeType: mime } : {}), videoBitsPerSecond }); const chunks: Blob[] = []; rec.ondataavailable = (e) => { if (e.data.size) chunks.push(e.data); }; const stopTracks = () => stream.getTracks().forEach((t) => t.stop()); let settled = false; // guard so onended + stop() don't double-resolve let onAutoEnd: (() => void) | null = null; const teardown = () => { removeListeners(); if (track) track.onended = null; stopTracks(); }; const build = (): CaptureResult => { const durationMs = Math.max(0, performance.now() - t0); const blob = new Blob(chunks, { type: rec.mimeType || "video/webm" }); const s = settings as MediaTrackSettings; const width = s.width || stream.getVideoTracks()[0]?.getSettings?.().width || window.innerWidth; const height = s.height || stream.getVideoTracks()[0]?.getSettings?.().height || window.innerHeight; // smooth the path, then inject each click's EXACT point as an anchor so the // cursor lands dead-on the target the instant a ripple fires (smoothing can // otherwise leave it a few px shy of where the click actually happened) const track = smoothCursor(cursor); for (const c of clicks) track.push({ t: Math.round(c.t), x: c.x, y: c.y }); track.sort((a, b) => a.t - b.t); const base: ScreencastData = { cursor: track, clicks, fps, source, drawCursor: animatedCursor, }; const screencast = withAutoZoom(base, { durationMs, style: opts.cameraStyle, ...opts.autoZoom }); return { blob, durationMs: Math.round(durationMs), width, height, screencast }; }; const stop = (): Promise => new Promise((resolve) => { if (settled) return; // already stopped (e.g. via the browser "Stop sharing" UI) settled = true; teardown(); const finish = () => resolve(build()); if (rec.state === "inactive") finish(); else { rec.onstop = finish; rec.stop(); } }); // user ends sharing via the browser's own UI → finalize exactly like an explicit stop if (track) { onAutoEnd = () => { if (!settled) void stop(); }; track.onended = onAutoEnd; } const cancel = () => { if (settled) return; settled = true; teardown(); if (rec.state !== "inactive") { rec.onstop = null; rec.stop(); } }; rec.start(); return { stop, cancel, stream }; }