import { useState, useRef, useEffect, useCallback } from "react"; import { Input, ALL_FORMATS, BlobSource, CanvasSink, Output, BufferTarget, Conversion, WebMOutputFormat, QUALITY_VERY_HIGH, } from "mediabunny"; import { FileVideo, Type, Download, Play, Pause, Plus, Trash2, X, Layers, } from "lucide-react"; import { pipeline } from "@huggingface/transformers"; interface TextElement { id: string; text: string; x: number; // absolute canvas pixels y: number; // absolute canvas pixels fontSize: number; color: string; fontFamily: string; bold: boolean; italic: boolean; strokeColor: string; strokeWidth: number; opacity: number; } interface VideoMeta { width: number; height: number; duration: number; fps: number; } type DragState = { on: boolean; mode: "move" | "resize"; id: string; ox: number; oy: number; startX: number; startY: number; startSize: number; didDrag: boolean; }; const uid = () => Math.random().toString(36).slice(2, 9); const FONTS = [ // Display / Heavy "Bebas Neue", "Anton", "Bungee", "Archivo Black", "Black Ops One", "Alfa Slab One", "Titan One", "Rubik Mono One", "Ultra", "Monoton", // Bold Sans "Oswald", "Teko", "Fjalla One", "Barlow Condensed", "Rajdhani", "Russo One", "Orbitron", "Michroma", "Chakra Petch", "Exo 2", // Serif / Elegant "Playfair Display", "Cinzel", "Cormorant Garamond", "Lora", "DM Serif Display", // Script / Handwriting "Pacifico", "Permanent Marker", "Caveat", "Dancing Script", "Sacramento", "Satisfy", "Great Vibes", "Lobster", // Fun / Decorative "Righteous", "Bangers", "Luckiest Guy", "Fredoka", "Passion One", "Press Start 2P", "Silkscreen", // Clean / Modern "Montserrat", "Raleway", "Poppins", "Space Grotesk", "Sora", // System "Impact", "Arial", ]; const HANDLE_R = 8; const PAD = 12; function fontStr(el: TextElement): string { return `${el.italic ? "italic " : ""}${el.bold ? "bold " : ""}${el.fontSize}px "${el.fontFamily}", sans-serif`; } type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; interface TextBounds { left: number; top: number; right: number; bottom: number; tw: number; } const getFitMeasureCtx = (() => { let ctx: CanvasRenderingContext2D | null = null; return () => { if (!ctx) { const canvas = document.createElement("canvas"); ctx = canvas.getContext("2d")!; } return ctx; }; })(); function getFloorTimestamp(timestamps: number[], time: number): number | null { if (!timestamps.length) return null; if (time < timestamps[0]) return timestamps[0]; let lo = 0; let hi = timestamps.length; while (lo < hi) { const mid = (lo + hi) >>> 1; if (timestamps[mid] <= time) lo = mid + 1; else hi = mid; } return timestamps[lo - 1] ?? null; } function measureTextBounds(ctx: Ctx2D, el: TextElement): TextBounds { ctx.font = fontStr(el); ctx.textAlign = "center"; ctx.textBaseline = "middle"; const tw = ctx.measureText(el.text).width; return { tw, left: el.x - tw / 2 - PAD, top: el.y - el.fontSize * 0.65 - PAD / 2, right: el.x + tw / 2 + PAD, bottom: el.y + el.fontSize * 0.65 + PAD / 2, }; } function drawTextLayer( ctx: Ctx2D, elements: TextElement[], selectedId?: string | null, ) { for (const el of elements) { ctx.font = fontStr(el); ctx.textAlign = "center"; ctx.textBaseline = "middle"; ctx.save(); ctx.globalAlpha = el.opacity; if (el.strokeWidth > 0) { ctx.strokeStyle = el.strokeColor; ctx.lineWidth = el.strokeWidth * 2; ctx.lineJoin = "round"; ctx.strokeText(el.text, el.x, el.y); } ctx.fillStyle = el.color; ctx.fillText(el.text, el.x, el.y); ctx.restore(); if (el.id === selectedId) { const b = measureTextBounds(ctx, el); ctx.save(); ctx.setLineDash([5, 4]); ctx.strokeStyle = "#818cf8"; ctx.lineWidth = 1.5; ctx.lineJoin = "round"; ctx.strokeRect(b.left, b.top, b.right - b.left, b.bottom - b.top); ctx.restore(); ctx.save(); ctx.beginPath(); ctx.arc(b.right, b.bottom, HANDLE_R, 0, Math.PI * 2); ctx.fillStyle = "#818cf8"; ctx.fill(); ctx.strokeStyle = "#fff"; ctx.lineWidth = 1.5; ctx.stroke(); ctx.restore(); } } } function computeFitFontSize( text: string, fontFamily: string, bold: boolean, canvasWidth: number, ): number { const ctx = getFitMeasureCtx(); const target = canvasWidth * 0.85; let lo = 10, hi = 600; while (hi - lo > 1) { const mid = Math.floor((lo + hi) / 2); ctx.font = `${bold ? "bold " : ""}${mid}px "${fontFamily}", sans-serif`; if (ctx.measureText(text).width <= target) lo = mid; else hi = mid; } return lo; } function computeAverageBrightness(canvas: HTMLCanvasElement): number { const ctx = canvas.getContext("2d")!; const data = ctx.getImageData(0, 0, canvas.width, canvas.height).data; let total = 0, count = 0; const step = Math.max(4, Math.floor(data.length / 40000)); for (let i = 0; i < data.length; i += step * 4) { total += data[i] * 0.299 + data[i + 1] * 0.587 + data[i + 2] * 0.114; count++; } return total / count; } function ProgressBar({ value }: { value: number }) { return (
); } export default function App() { const pipelineRef = useRef(null); const [modelReady, setModelReady] = useState(false); const pendingRef = useRef<{ file: File; meta: VideoMeta } | null>(null); const [videoFile, setVideoFile] = useState(null); const [videoObjectUrl, setVideoObjectUrl] = useState(""); const [meta, setMeta] = useState(null); const [curTime, setCurTime] = useState(0); const [isPlaying, setIsPlaying] = useState(false); const [procStatus, setProcStatus] = useState<"idle" | "running" | "done">( "idle", ); const [procProgress, setProcProgress] = useState(0); const [procMsg, setProcMsg] = useState(""); const fgRef = useRef>(new Map()); const tsRef = useRef([]); const [texts, setTexts] = useState([]); const [selectedId, setSelectedId] = useState(null); const [textPanelOpen, setTextPanelOpen] = useState(false); const [smartColor, setSmartColor] = useState("#ffffff"); const [renderStatus, setRenderStatus] = useState<"idle" | "running">("idle"); const [renderProgress, setRenderProgress] = useState(0); const canvasRef = useRef(null); const videoRef = useRef(null); const frameCacheRef = useRef(null); const lastDrawnTimeRef = useRef(0); const dragRef = useRef({ on: false, mode: "move", id: "", ox: 0, oy: 0, startX: 0, startY: 0, startSize: 0, didDrag: false, }); const rafRef = useRef(0); const textsRef = useRef(texts); textsRef.current = texts; const selectedIdRef = useRef(selectedId); selectedIdRef.current = selectedId; useEffect(() => { if (pipelineRef.current) return; (async () => { try { pipelineRef.current = await pipeline( "background-removal", "onnx-community/BEN2-ONNX", { device: "webgpu" }, ); } catch (e) { console.error("Model load failed:", e); alert("Failed to load background removal model. See console for details."); return; } setModelReady(true); })(); }, []); useEffect(() => { if (modelReady && pendingRef.current) { const { file, meta: m } = pendingRef.current; pendingRef.current = null; processFrames(file, m); } }, [modelReady]); useEffect(() => { const canvas = canvasRef.current; if (!canvas) return; canvas.width = meta?.width ?? 1280; canvas.height = meta?.height ?? 720; }, [meta]); const findFgFrame = useCallback((t: number): HTMLCanvasElement | null => { const key = getFloorTimestamp(tsRef.current, t); return key == null ? null : (fgRef.current.get(key) ?? null); }, []); const drawCanvas = useCallback(() => { const canvas = canvasRef.current; const vid = videoRef.current; if (!canvas || canvas.width === 0) return; const ctx = canvas.getContext("2d"); if (!ctx) return; const w = canvas.width, h = canvas.height; ctx.clearRect(0, 0, w, h); let drawnTime = lastDrawnTimeRef.current; if (vid && vid.readyState >= 2) { ctx.drawImage(vid, 0, 0, w, h); drawnTime = vid.currentTime; lastDrawnTimeRef.current = drawnTime; if (!frameCacheRef.current) frameCacheRef.current = document.createElement("canvas"); const fc = frameCacheRef.current; if (fc.width !== w || fc.height !== h) { fc.width = w; fc.height = h; } fc.getContext("2d")!.drawImage(vid, 0, 0, w, h); } else if (frameCacheRef.current) { ctx.drawImage(frameCacheRef.current, 0, 0); } else { ctx.fillStyle = "#0a0a0a"; ctx.fillRect(0, 0, w, h); } drawTextLayer(ctx, textsRef.current, selectedIdRef.current); const fg = findFgFrame(drawnTime); if (fg) ctx.drawImage(fg, 0, 0, w, h); }, [findFgFrame]); useEffect(() => { const loop = () => { drawCanvas(); rafRef.current = requestAnimationFrame(loop); }; rafRef.current = requestAnimationFrame(loop); return () => cancelAnimationFrame(rafRef.current); }, [drawCanvas]); function onVideoReady() { const vid = videoRef.current; if (!vid || !vid.videoWidth) return; const c = document.createElement("canvas"); c.width = vid.videoWidth; c.height = vid.videoHeight; c.getContext("2d")!.drawImage(vid, 0, 0, c.width, c.height); const brightness = computeAverageBrightness(c); setSmartColor(brightness > 128 ? "#1a1a2e" : "#ffffff"); } function onTimeUpdate() { const vid = videoRef.current; if (vid) setCurTime(vid.currentTime); } function togglePlay() { const vid = videoRef.current; if (!vid) return; if (vid.paused) { void vid.play(); setIsPlaying(true); } else { vid.pause(); setIsPlaying(false); } } async function handleFile(file: File) { if (videoObjectUrl) URL.revokeObjectURL(videoObjectUrl); const url = URL.createObjectURL(file); setVideoObjectUrl(url); setVideoFile(file); setMeta(null); setCurTime(0); setIsPlaying(false); setProcStatus("idle"); setProcProgress(0); setProcMsg(""); fgRef.current.clear(); tsRef.current = []; setRenderStatus("idle"); setRenderProgress(0); frameCacheRef.current = null; setTexts([]); setSelectedId(null); try { const inp = new Input({ formats: ALL_FORMATS, source: new BlobSource(file), }); const vt = await inp.getPrimaryVideoTrack(); if (!vt) { alert("No video track found"); return; } const duration = await inp.computeDuration(); const stats = await vt.computePacketStats(60); const newMeta: VideoMeta = { width: vt.displayWidth, height: vt.displayHeight, duration, fps: stats.averagePacketRate, }; setMeta(newMeta); if (modelReady && pipelineRef.current) { processFrames(file, newMeta); } else { pendingRef.current = { file, meta: newMeta }; } } catch (e) { console.error(e); alert("Could not read video: " + e); } } async function processFrames(fileArg?: File, metaArg?: VideoMeta) { const file = fileArg ?? videoFile; const m = metaArg ?? meta; if (!file || !m || !pipelineRef.current) return; setProcStatus("running"); setProcProgress(0); fgRef.current.clear(); tsRef.current = []; try { const inp = new Input({ formats: ALL_FORMATS, source: new BlobSource(file), }); const vt = await inp.getPrimaryVideoTrack(); if (!vt) throw new Error("No video track"); const sink = new CanvasSink(vt, { width: m.width, height: m.height, fit: "fill", }); const estimatedTotal = Math.ceil(m.duration * m.fps); let count = 0; for await (const { canvas, timestamp } of sink.canvases()) { setProcMsg(`Removing background · frame ${count + 1}`); const result = await pipelineRef.current(canvas); const fgCanvas: HTMLCanvasElement = result.toCanvas(); fgRef.current.set(timestamp, fgCanvas); tsRef.current.push(timestamp); count++; setProcProgress(Math.min((count / estimatedTotal) * 100, 99)); } tsRef.current.sort((a, b) => a - b); setProcStatus("done"); setProcProgress(100); setProcMsg(`${count} frames processed`); } catch (e) { console.error(e); setProcStatus("idle"); setProcMsg(""); alert("Processing error: " + e); } } async function renderVideo() { if (!videoFile || !meta || renderStatus === "running") return; setRenderStatus("running"); setRenderProgress(0); try { const inp = new Input({ formats: ALL_FORMATS, source: new BlobSource(videoFile), }); const out = new Output({ format: new WebMOutputFormat(), target: new BufferTarget(), }); const textSnap = [...texts]; const fgMap = fgRef.current; const tsList = [...tsRef.current]; const metaSnap = meta; function nearestFg(t: number): HTMLCanvasElement | null { const key = getFloorTimestamp(tsList, t); return key == null ? null : (fgMap.get(key) ?? null); } let offscreen: OffscreenCanvas | null = null; let octx: OffscreenCanvasRenderingContext2D | null = null; const conv = await Conversion.init({ input: inp, output: out, video: { forceTranscode: true, codec: "vp9", bitrate: QUALITY_VERY_HIGH, process: (sample) => { if (!offscreen) { offscreen = new OffscreenCanvas( sample.displayWidth, sample.displayHeight, ); octx = offscreen.getContext("2d")!; } const ctx = octx!; ctx.clearRect(0, 0, offscreen.width, offscreen.height); sample.draw(ctx, 0, 0); const scale = offscreen.width / metaSnap.width; const scaledTexts = scale === 1 ? textSnap : textSnap.map((t) => ({ ...t, x: t.x * scale, y: t.y * scale, fontSize: Math.round(t.fontSize * scale), strokeWidth: t.strokeWidth * scale, })); drawTextLayer(ctx, scaledTexts, null); const fg = nearestFg(sample.timestamp); if (fg) ctx.drawImage(fg, 0, 0, offscreen.width, offscreen.height); return offscreen; }, }, }); conv.onProgress = (p) => setRenderProgress(Math.round(p * 100)); await conv.execute(); const buf = out.target.buffer!; const blob = new Blob([buf], { type: "video/webm" }); const url = URL.createObjectURL(blob); // Trigger download immediately const a = document.createElement("a"); a.href = url; a.download = "text-behind-video.webm"; document.body.appendChild(a); a.click(); document.body.removeChild(a); URL.revokeObjectURL(url); setRenderStatus("idle"); } catch (e) { console.error(e); alert("Render error: " + e); setRenderStatus("idle"); } } function getPointerPos( e: React.PointerEvent, ): [number, number] { const canvas = e.currentTarget; const r = canvas.getBoundingClientRect(); const sx = canvas.width / r.width; const sy = canvas.height / r.height; return [(e.clientX - r.left) * sx, (e.clientY - r.top) * sy]; } function onPointerDown(e: React.PointerEvent) { if (procStatus === "running" || renderStatus === "running") return; const [mx, my] = getPointerPos(e); const canvas = e.currentTarget; const ctx = canvas.getContext("2d")!; if (selectedId) { const selEl = texts.find((t) => t.id === selectedId); if (selEl) { const b = measureTextBounds(ctx, selEl); const dist = Math.sqrt((mx - b.right) ** 2 + (my - b.bottom) ** 2); if (dist <= HANDLE_R + 4) { canvas.setPointerCapture(e.pointerId); dragRef.current = { on: true, mode: "resize", id: selEl.id, ox: mx, oy: my, startX: selEl.x, startY: selEl.y, startSize: selEl.fontSize, didDrag: false, }; return; } } } for (let i = texts.length - 1; i >= 0; i--) { const el = texts[i]; const b = measureTextBounds(ctx, el); if (mx >= b.left && mx <= b.right && my >= b.top && my <= b.bottom) { canvas.setPointerCapture(e.pointerId); setSelectedId(el.id); dragRef.current = { on: true, mode: "move", id: el.id, ox: mx, oy: my, startX: el.x, startY: el.y, startSize: el.fontSize, didDrag: false, }; return; } } setSelectedId(null); } function onPointerMove(e: React.PointerEvent) { const dr = dragRef.current; if (!dr.on) return; const [mx, my] = getPointerPos(e); if (!dr.didDrag) { const dist = Math.abs(mx - dr.ox) + Math.abs(my - dr.oy); if (dist < 4) return; dr.didDrag = true; } if (dr.mode === "move") { setTexts((prev) => prev.map((t) => t.id === dr.id ? { ...t, x: dr.startX + (mx - dr.ox), y: dr.startY + (my - dr.oy) } : t, ), ); } else { const delta = (mx - dr.ox + my - dr.oy) / 2; const newSize = Math.max( 10, Math.min(600, Math.round(dr.startSize + delta * 0.4)), ); setTexts((prev) => prev.map((t) => (t.id === dr.id ? { ...t, fontSize: newSize } : t)), ); } } function onPointerUp() { const dr = dragRef.current; if (dr.on && !dr.didDrag && dr.id) { setTextPanelOpen(true); } dr.on = false; } function addText() { const w = canvasRef.current?.width ?? meta?.width ?? 1280; const h = canvasRef.current?.height ?? meta?.height ?? 720; const fitSize = computeFitFontSize("EDIT TEXT", "Archivo Black", false, w); const el: TextElement = { id: uid(), text: "EDIT TEXT", x: w / 2, y: h / 2, fontSize: fitSize, color: smartColor, fontFamily: "Archivo Black", bold: false, italic: false, strokeColor: "#000000", strokeWidth: 0, opacity: 1, }; setTexts((prev) => [...prev, el]); setSelectedId(el.id); setTextPanelOpen(true); } function updateText(id: string, patch: Partial) { setTexts((prev) => prev.map((t) => (t.id === id ? { ...t, ...patch } : t))); } const hasVideo = !!videoFile && !!meta; const isProcessing = procStatus === "running"; const isExporting = renderStatus === "running"; const canInteract = hasVideo && !isProcessing && !isExporting; const activeText = texts.find((t) => t.id === selectedId) ?? null; return (