/** * AI Video Generation Benchmark — a static "display" space. * * Fetches the canonical benchmark JSON from mesmer.tools once at load and renders * a leaderboard + per-prompt video gallery. No input form, no per-visitor API * calls — the data is regenerated whenever the main site redeploys, so this page * stays current on its own. */ import { mountChrome, el } from "./shared/ui.js"; import { fetchData } from "./shared/api-client.js"; import { DATA, ENDPOINTS, siteUrl } from "./shared/config.js"; const FULL_BENCHMARK_URL = siteUrl(ENDPOINTS.benchmark.fullToolPath); // mesmer.tools/benchmarks/ai-video-generation /* --- Formatting helpers --------------------------------------------------- */ function fmtSeconds(ms) { return ms == null ? "—" : `${(ms / 1000).toFixed(1)}s`; } function fmtCodeKB(chars) { return chars == null ? "—" : `${(chars / 1000).toFixed(1)} KB`; } function fmtCost(usd, estimate) { if (usd == null) return "—"; return `${estimate ? "~$" : "$"}${usd.toFixed(2)}`; } function fmtDate(iso) { const d = new Date(iso); if (Number.isNaN(d.getTime())) return iso || "—"; return d.toLocaleDateString("en-US", { year: "numeric", month: "short", day: "numeric" }); } function runnerLabel(runner) { return runner === "claude-code-subagent" ? "Claude Code" : "OpenRouter"; } function truncate(text, max = 90) { if (!text) return ""; return text.length > max ? `${text.slice(0, max - 1).trimEnd()}…` : text; } /* --- Page shell ----------------------------------------------------------- */ const main = el("main", { className: "bm-main" }); main.appendChild(el("div", { className: "bm-state" }, el("span", { className: "ms-spinner", "aria-hidden": "true" }), el("span", { text: "Loading benchmark…" }), )); // Insert main first, then mount the shared chrome so ordering is header → main → footer. const root = document.getElementById("app") || document.body; root.appendChild(main); mountChrome("benchmark"); /* --- Fetch + render ------------------------------------------------------- */ fetchData(DATA.videoBenchmark) .then((data) => { if (!data || !Array.isArray(data.summaries) || !Array.isArray(data.prompts)) { throw new Error("The benchmark data was empty or in an unexpected shape."); } renderBenchmark(data); }) .catch((err) => { main.replaceChildren( el("div", { className: "bm-state is-error" }, el("strong", { text: "Couldn't load the benchmark." }), el("span", { text: (err && err.message) || "Please try again in a moment." }), el("a", { className: "bm-retry", href: FULL_BENCHMARK_URL, target: "_blank", rel: "noopener", text: "See it on mesmer.tools →" }), ), ); }); function renderBenchmark(data) { const { generatedAt, models = [], prompts = [], summaries = [], sections = [], totals = {} } = data; const ranking = Array.isArray(data.ranking) ? data.ranking : []; const modelBySlug = new Map(models.map((m) => [m.slug, m])); const rankOf = (slug) => { const i = ranking.indexOf(slug); return i >= 0 ? i + 1 : null; }; const frag = document.createDocumentFragment(); frag.appendChild(hero(generatedAt, models, prompts, totals)); frag.appendChild(leaderboard(summaries)); frag.appendChild(promptGallery(prompts, sections, modelBySlug, rankOf)); frag.appendChild(ctaCard()); main.replaceChildren(frag); } /* --- Hero ----------------------------------------------------------------- */ function hero(generatedAt, models, prompts, totals) { return el("section", { className: "bm-hero" }, el("div", { className: "bm-hero-emoji", text: "🏁" }), el("h1", { className: "bm-title", text: "AI Video Generation Benchmark" }), el("p", { className: "bm-subtitle", text: `Which LLM writes the best motion-graphics code? ${models.length} models, ${prompts.length} briefs, rendered to video.` }), el("div", { className: "bm-method" }, el("span", { className: "bm-method-icon", text: "🧪" }), el("p", { className: "bm-method-body" }, el("strong", { text: "How it works. " }), "Every model got the same creative briefs and was asked to generate animation code, which was then compiled and rendered to video — the same pipeline for all, with no hand-fixing in between. Failed cards are the model's own: the code either didn't generate or crashed while rendering. The ranking below is editorial, judged by hand on output quality.", ), ), el("p", { className: "bm-updated", text: generatedAt ? `Updated ${fmtDate(generatedAt)} · ${totals.rendered ?? 0}/${totals.total ?? 0} videos rendered` : `${totals.rendered ?? 0}/${totals.total ?? 0} videos rendered · live from mesmer.tools` }), ); } /* --- Leaderboard ---------------------------------------------------------- */ function leaderboard(summaries) { const head = el("tr", {}, el("th", { text: "Rank" }), el("th", { text: "Model" }), el("th", { text: "Runner" }), el("th", { text: "Rendered" }), el("th", { text: "Avg time" }), el("th", { text: "Avg code" }), el("th", { text: "Cost" }), el("th", { text: "Tries" }), ); const rows = summaries.map((s, i) => { const m = s.model || {}; const isClaude = m.runner === "claude-code-subagent"; const renderedCls = s.rendered === s.total ? "is-full" : s.rendered === 0 ? "is-zero" : "is-partial"; return el("tr", {}, el("td", { className: "bm-rank", text: String(i + 1) }), el("td", {}, el("div", { className: "bm-model-name", text: m.displayName || m.slug || "—" }), el("div", { className: "bm-model-prov", text: m.provider || "" }), ), el("td", {}, el("span", { className: `bm-badge ${isClaude ? "is-cc" : "is-or"}`, text: runnerLabel(m.runner) }), ), el("td", {}, el("span", { className: `bm-rendered ${renderedCls}`, text: `${s.rendered}/${s.total}` }), ), el("td", { className: "bm-num", text: fmtSeconds(s.avgGenerationMs) }), el("td", { className: "bm-num", text: fmtCodeKB(s.avgCodeChars) }), el("td", { className: "bm-num", text: fmtCost(s.totalCostUsd, isClaude) }), el("td", { className: "bm-num", text: s.totalAttempts != null ? String(s.totalAttempts) : "—" }), ); }); return el("section", { className: "bm-section" }, el("h2", { className: "bm-h2", text: "Leaderboard" }), el("p", { className: "bm-h2-note", text: "Ranked by hand after watching every model's videos. Claude models ran through Claude Code, so their gen times are omitted and cost is an estimate (~)." }), el("p", { className: "bm-scroll-hint", text: "← swipe the table for all columns →" }), el("div", { className: "bm-table-wrap" }, el("table", { className: "bm-table" }, el("thead", {}, head), el("tbody", {}, ...rows), ), ), ); } /* --- Per-prompt video gallery -------------------------------------------- */ function promptGallery(prompts, sections, modelBySlug, rankOf) { const sectionByPrompt = new Map(sections.map((s) => [s.promptId, s])); const wrap = el("section", { className: "bm-section" }, el("h2", { className: "bm-h2", text: "Every result, brief by brief" }), el("p", { className: "bm-h2-note", text: "Same prompt, every model. Best-ranked first; failures last." }), ); for (const prompt of prompts) { const section = sectionByPrompt.get(prompt.id); if (!section || !Array.isArray(section.runs)) continue; const aspect = prompt.aspectRatio || "16:9"; const grid = el("div", { className: "bm-grid", dataset: { aspect } }); for (const run of section.runs) { grid.appendChild(videoCard(run, prompt, modelBySlug, rankOf)); } wrap.appendChild( el("div", { className: "bm-prompt" }, el("div", { className: "bm-prompt-head" }, el("h3", { className: "bm-prompt-title", text: prompt.title || prompt.id }), el("span", { className: "bm-prompt-spec", text: `${prompt.aspectRatio} · ${prompt.fps}fps` }), ), el("details", { className: "bm-prompt-details" }, el("summary", { text: "Show the full brief" }), el("div", { className: "bm-prompt-text", text: prompt.prompt || "" }), ), grid, ), ); } return wrap; } function videoCard(run, prompt, modelBySlug, rankOf) { const m = modelBySlug.get(run.modelSlug) || {}; const name = m.displayName || run.modelSlug; const rank = rankOf(run.modelSlug); const sub = [rank ? `#${rank}` : null, m.provider].filter(Boolean).join(" · "); const aspect = (prompt.aspectRatio || "16:9").replace(":", " / "); if (run.rendered && run.videoUrl) { const metaParts = []; if (run.generationMs != null) metaParts.push(`${(run.generationMs / 1000).toFixed(1)}s gen`); if (run.codeChars != null) metaParts.push(`${(run.codeChars / 1000).toFixed(1)} KB code`); return el("div", { className: "bm-card" }, el("video", { src: run.videoUrl, controls: "", muted: "", loop: "", playsinline: "", preload: "metadata", width: String(prompt.width || ""), height: String(prompt.height || ""), "aria-label": `${name} — ${prompt.title}`, }), el("div", { className: "bm-card-name", text: name }), el("div", { className: "bm-card-sub", text: sub }), metaParts.length ? el("div", { className: "bm-card-meta", text: metaParts.join(" · ") }) : null, ); } // Failed run — dashed placeholder sized to the brief's aspect ratio. const reason = run.error ? truncate(run.error) : "Code didn't compile"; return el("div", { className: "bm-card is-fail" }, el("div", { className: "bm-fail", style: { aspectRatio: aspect }, text: run.codeUrl ? "Code didn't render" : "No usable code produced" }), el("div", { className: "bm-card-name", text: name }), el("div", { className: "bm-card-sub", text: sub }), el("div", { className: "bm-card-meta", text: reason }), ); } /* --- CTA ------------------------------------------------------------------ */ function ctaCard() { return el("section", { className: "bm-cta" }, el("h3", { text: "Want to compare them side by side?" }), el("p", { text: "The full interactive benchmark adds a lightbox, side-by-side compare, and the raw generated code for every model." }), el("a", { className: "bm-cta-btn", href: FULL_BENCHMARK_URL, target: "_blank", rel: "noopener", text: "See the full interactive benchmark with side-by-side compare →" }), ); }