eldouma-graphics / src /lib /screenCapture.ts
Moeeldouma's picture
Deploy Eldouma Graphics β€” Docker Space, bring-your-own-key demo
39d98a0 verified
Raw
History Blame Contribute Delete
12.5 kB
// ─────────────────────────────────────────────────────────────────────────────
// Browser-side screen-capture for the "explainer zoom" feature.
//
// Records the screen (preferring THIS tab) via getDisplayMedia while sampling the
// cursor track + click events off the live DOM, then assembles a ScreencastData
// bundle (cursor / clicks / cam) that rides alongside the recorded webm so the
// renderer can drive the auto-zoom camera. See screencast.ts for the data model
// and screencastZoom.ts for the camera engine.
//
// Pure DOM / Media APIs β€” no React. Capture-time only, so the non-determinism
// (performance.now / getBoundingClientRect) is fine; the renderer is the part that
// must stay frame-deterministic.
// ─────────────────────────────────────────────────────────────────────────────
import type {
ScreencastData, ScreencastPoint, ScreencastClick, ScreencastRect,
} from "./screencast";
import { withAutoZoom, type AutoZoomOpts, type ZoomStyle } from "./screencastZoom";
const clamp01 = (v: number) => Math.min(1, Math.max(0, v));
export interface CaptureResult {
blob: Blob;
durationMs: number;
width: number;
height: number;
screencast: ScreencastData;
}
export interface CaptureController {
stop: () => Promise<CaptureResult>;
cancel: () => void;
stream: MediaStream;
}
// Capture quality presets. getDisplayMedia can't exceed the actual surface's pixel
// count, but requesting a high `ideal` makes Chrome capture at the surface's full
// (DPR-scaled) resolution. Recording ABOVE the composition size is what keeps the
// footage crisp when the explainer camera zooms in (a 2Γ— zoom on 4K source still
// has full-HD detail; a 2Γ— zoom on 1080 source is soft).
export type CaptureQuality = "1080p" | "1440p" | "4k";
const QUALITY: Record<CaptureQuality, { w: number; h: number; bitrate: number }> = {
"1080p": { w: 1920, h: 1080, bitrate: 8_000_000 },
"1440p": { w: 2560, h: 1440, bitrate: 18_000_000 },
"4k": { w: 3840, h: 2160, bitrate: 44_000_000 },
};
export interface ScreenCaptureOpts {
frameRate?: number; // requested capture frame rate (default 30; 60 for buttery motion)
quality?: CaptureQuality; // resolution + bitrate preset (default "1440p")
animatedCursor?: boolean; // suppress the OS cursor at capture + render our own sprite (default true)
cameraStyle?: ZoomStyle; // how eagerly the explainer camera zooms (default "calm")
autoZoom?: AutoZoomOpts; // fine-grain tuning passed through to withAutoZoom (overrides cameraStyle)
}
// Map getDisplayMedia's displaySurface to our coarse ScreencastData.source bucket.
function mapSource(surface?: string): ScreencastData["source"] {
if (surface === "browser") return "tab";
if (surface === "window") return "window";
return "screen"; // "monitor" / unknown
}
// Walk up from a hit-tested element to the nearest "meaningful" target (a thing a
// user would think they clicked) β€” so the framing rect is the button/link, not a
// stray text node. Capped to a few levels so we never balloon to a layout wrapper.
function meaningfulEl(start: Element | null): Element | null {
let el: Element | null = start;
for (let i = 0; el && i < 4; i++) {
const tag = el.tagName.toLowerCase();
if (
el.getAttribute("aria-label") ||
el.getAttribute("role") === "button" ||
tag === "button" || tag === "a" || tag === "input" ||
el.getAttribute("data-testid")
) return el;
el = el.parentElement;
}
return start;
}
function elLabel(el: Element): string {
return (
el.getAttribute("aria-label") ||
el.getAttribute("title") ||
(el.textContent || "").trim().slice(0, 40) ||
el.tagName.toLowerCase()
);
}
// ── cursor smoothing: a 1€ filter ────────────────────────────────────────────
// EMA lags behind fast motion (the cursor visibly trails the real pointer). The 1€
// filter (Casiez et al.) raises its cutoff with speed: heavy smoothing when the
// cursor is slow/still (kills jitter), almost no smoothing when it's moving fast
// (no lag) β€” exactly the behavior a cursor wants. Then we downsample to ~targetHz.
const lowpass = (a: number, x: number, prev: number) => a * x + (1 - a) * prev;
const alphaFor = (cutoff: number, dt: number) => { const tau = 1 / (2 * Math.PI * cutoff); return 1 / (1 + tau / dt); };
function smoothCursor(raw: ScreencastPoint[], targetHz = 30): ScreencastPoint[] {
if (raw.length === 0) return raw;
const sorted = [...raw].sort((a, b) => a.t - b.t);
const minCutoff = 1.6, beta = 0.45, dCutoff = 1.0; // tuned for 0..1 coordinate space
let xP = sorted[0].x, yP = sorted[0].y, dxP = 0, dyP = 0, tP = sorted[0].t;
const filtered: ScreencastPoint[] = [{ t: Math.round(tP), x: clamp01(xP), y: clamp01(yP) }];
for (let i = 1; i < sorted.length; i++) {
const p = sorted[i];
const dt = Math.max(1, p.t - tP) / 1000; // seconds
const dx = (p.x - xP) / dt, dy = (p.y - yP) / dt;
const ad = alphaFor(dCutoff, dt);
dxP = lowpass(ad, dx, dxP); dyP = lowpass(ad, dy, dyP);
const ax = alphaFor(minCutoff + beta * Math.abs(dxP), dt);
const ay = alphaFor(minCutoff + beta * Math.abs(dyP), dt);
xP = lowpass(ax, p.x, xP); yP = lowpass(ay, p.y, yP); tP = p.t;
filtered.push({ t: Math.round(p.t), x: clamp01(xP), y: clamp01(yP) });
}
// downsample to ~targetHz, always keeping the first & last
const minGap = 1000 / targetHz;
const out: ScreencastPoint[] = [filtered[0]];
for (const p of filtered) if (p.t - out[out.length - 1].t >= minGap) out.push(p);
const last = filtered[filtered.length - 1];
if (out[out.length - 1].t < last.t) out.push(last);
return out;
}
export async function startScreenCapture(opts: ScreenCaptureOpts = {}): Promise<CaptureController> {
const fps = opts.frameRate ?? 30;
const q = QUALITY[opts.quality ?? "1440p"];
const animatedCursor = opts.animatedCursor !== false; // default true
const stream = await navigator.mediaDevices.getDisplayMedia({
video: {
frameRate: { ideal: fps, max: fps },
width: { ideal: q.w },
height: { ideal: q.h },
// suppress the OS cursor in the recorded pixels so it doesn't double up with our
// animated sprite (best-effort: some browsers ignore it β€” hence the render-side flag)
cursor: animatedCursor ? "never" : "always",
},
audio: false,
// captures THIS tab when the user agrees β†’ DOM rects map to captured pixels
preferCurrentTab: true,
} as DisplayMediaStreamOptions);
const track = stream.getVideoTracks()[0];
const settings = track?.getSettings?.() ?? {};
const source = mapSource((settings as MediaTrackSettings & { displaySurface?: string }).displaySurface);
const isTab = source === "tab";
// ── metadata sampling ──────────────────────────────────────────────────────
const t0 = performance.now();
const cursor: ScreencastPoint[] = [];
const clicks: ScreencastClick[] = [];
const norm = (e: { clientX: number; clientY: number }) => ({
x: clamp01(e.clientX / window.innerWidth),
y: clamp01(e.clientY / window.innerHeight),
});
let lastMove = 0;
const onMove = (e: PointerEvent) => {
const now = performance.now();
if (now - lastMove < 33) return; // ~30Hz throttle (smoothed/downsampled later)
lastMove = now;
const { x, y } = norm(e);
cursor.push({ t: now - t0, x, y });
};
const recordClick = (e: PointerEvent | MouseEvent, kind: ScreencastClick["kind"]) => {
const { x, y } = norm(e);
const click: ScreencastClick = { t: performance.now() - t0, x, y, kind };
// Only attach element geometry when capturing this tab β€” otherwise the captured
// pixels are a different surface and DOM coords wouldn't map reliably.
if (isTab) {
const hit = document.elementFromPoint(e.clientX, e.clientY);
const el = meaningfulEl(hit);
if (el) {
const r = el.getBoundingClientRect();
const W = window.innerWidth, H = window.innerHeight;
const rect: ScreencastRect = {
x: clamp01(r.left / W), y: clamp01(r.top / H),
w: clamp01(r.width / W), h: clamp01(r.height / H),
};
click.rect = rect;
click.label = elLabel(el);
}
}
clicks.push(click);
};
const onDown = (e: PointerEvent) => recordClick(e, "click");
const onDbl = (e: MouseEvent) => recordClick(e, "dblclick");
const onCtx = (e: MouseEvent) => recordClick(e, "contextmenu");
// capture phase + passive so we observe without interfering with the page
const listenOpts: AddEventListenerOptions = { capture: true, passive: true };
window.addEventListener("pointermove", onMove, listenOpts);
window.addEventListener("pointerdown", onDown, listenOpts);
window.addEventListener("dblclick", onDbl, listenOpts);
window.addEventListener("contextmenu", onCtx, listenOpts);
const removeListeners = () => {
window.removeEventListener("pointermove", onMove, listenOpts);
window.removeEventListener("pointerdown", onDown, listenOpts);
window.removeEventListener("dblclick", onDbl, listenOpts);
window.removeEventListener("contextmenu", onCtx, listenOpts);
};
// ── recorder ────────────────────────────────────────────────────────────────
const mime = ["video/webm;codecs=vp9", "video/webm;codecs=vp8", "video/webm"]
.find((m) => MediaRecorder.isTypeSupported?.(m)) || "";
// scale the target bitrate with frame rate so 60fps stays as crisp as 30fps
const videoBitsPerSecond = Math.round(q.bitrate * (fps / 30));
const rec = new MediaRecorder(stream, { ...(mime ? { mimeType: mime } : {}), videoBitsPerSecond });
const chunks: Blob[] = [];
rec.ondataavailable = (e) => { if (e.data.size) chunks.push(e.data); };
const stopTracks = () => stream.getTracks().forEach((t) => t.stop());
let settled = false; // guard so onended + stop() don't double-resolve
let onAutoEnd: (() => void) | null = null;
const teardown = () => {
removeListeners();
if (track) track.onended = null;
stopTracks();
};
const build = (): CaptureResult => {
const durationMs = Math.max(0, performance.now() - t0);
const blob = new Blob(chunks, { type: rec.mimeType || "video/webm" });
const s = settings as MediaTrackSettings;
const width = s.width || stream.getVideoTracks()[0]?.getSettings?.().width || window.innerWidth;
const height = s.height || stream.getVideoTracks()[0]?.getSettings?.().height || window.innerHeight;
// smooth the path, then inject each click's EXACT point as an anchor so the
// cursor lands dead-on the target the instant a ripple fires (smoothing can
// otherwise leave it a few px shy of where the click actually happened)
const track = smoothCursor(cursor);
for (const c of clicks) track.push({ t: Math.round(c.t), x: c.x, y: c.y });
track.sort((a, b) => a.t - b.t);
const base: ScreencastData = {
cursor: track,
clicks,
fps,
source,
drawCursor: animatedCursor,
};
const screencast = withAutoZoom(base, { durationMs, style: opts.cameraStyle, ...opts.autoZoom });
return { blob, durationMs: Math.round(durationMs), width, height, screencast };
};
const stop = (): Promise<CaptureResult> => new Promise((resolve) => {
if (settled) return; // already stopped (e.g. via the browser "Stop sharing" UI)
settled = true;
teardown();
const finish = () => resolve(build());
if (rec.state === "inactive") finish();
else { rec.onstop = finish; rec.stop(); }
});
// user ends sharing via the browser's own UI β†’ finalize exactly like an explicit stop
if (track) {
onAutoEnd = () => { if (!settled) void stop(); };
track.onended = onAutoEnd;
}
const cancel = () => {
if (settled) return;
settled = true;
teardown();
if (rec.state !== "inactive") { rec.onstop = null; rec.stop(); }
};
rec.start();
return { stop, cancel, stream };
}