/* Roblox LLM Leaderboard: table + charts over data/leaderboard.json */
"use strict";
// Torpedo Software viz palette (--viz-*); Torpedo itself wears the brand purple.
const PROVIDER_COLORS = {
OpenAI: "#0FA27E",
Anthropic: "#CC785C",
Google: "#34A853",
DeepSeek: "#082296",
Alibaba: "#FF6701",
Meta: "#0081FB",
xAI: "#B6B7BA",
"Z.ai": "#427182",
"Moonshot AI": "#A5B6C9",
"Mistral AI": "#FD5D21",
};
// Provider logos ride the scatter points. Every file under assets/providers is a
// white-on-transparent glyph, so a marker is a disc in the provider's color with
// the glyph tinted for contrast on top. Torpedo's brand mark is already a
// finished circular badge, so it is drawn edge to edge instead.
const PROVIDER_LOGOS = {
OpenAI: "assets/providers/OpenAI.svg",
Anthropic: "assets/providers/Anthropic.svg",
Google: "assets/providers/Google.svg",
DeepSeek: "assets/providers/Deepseek.svg",
Alibaba: "assets/providers/Alibaba.svg",
Meta: "assets/providers/Meta.svg",
xAI: "assets/providers/xAI.svg",
"Z.ai": "assets/providers/Z-ai.svg",
"Moonshot AI": "assets/providers/MoonshotAI.svg",
"Mistral AI": "assets/providers/Mistral.svg",
"Torpedo Software": "assets/brand/logo.svg",
};
const FULL_BLEED_LOGOS = new Set(["Torpedo Software"]);
const MARKER_SIZE = 18; // marker diameter, CSS px
const MARKER_HOVER_SIZE = 22;
const MARKER_RING = 1.5; // matches the point border the markers replace
const GLYPH_INSET = 0.56; // glyph box as a share of the disc
// Canvas text has no hinting or subpixel AA to lean on, so a chart drawn at the
// screen's own pixel ratio reads softer than the DOM around it, and on the
// fractional ratios Windows display scaling hands out, hairline gridlines land
// between device pixels. Drawing at double density and letting the compositor
// scale down is plain supersampling. Past 2x the screen is dense enough that
// the extra fill buys nothing, and 3x is where the cost stops being free.
const renderScale = () => {
const dpr = window.devicePixelRatio || 1;
return Math.min(3, dpr < 2 ? dpr * 2 : dpr);
};
// Marker bitmaps have to out-resolve the canvas they get drawn into.
const markerScale = () => Math.max(3, renderScale());
const logoImages = {}; // provider -> decoded
, absent if the file failed
const markerImages = {}; // `${provider}|${size}` -> composed marker
function cssVar(name) {
return getComputedStyle(document.documentElement)
.getPropertyValue(name)
.trim();
}
function withAlpha(color, alpha) {
if (color.startsWith("#") && color.length === 7) {
const a = Math.round(alpha * 255)
.toString(16)
.padStart(2, "0");
return color + a;
}
return color;
}
const state = {
data: null,
columnSet: "overview",
difficulty: "overall",
qaSplit: "overall",
sortKey: "overall",
sortDir: -1,
search: "",
weightsFilter: "all", // "all" | "open" (published weights, i.e. self-hostable)
expandedId: null,
difficultyMetric: "fullySolved",
failureBench: "leetcode",
topicSplit: "overall",
};
const charts = {}; // canvas id -> Chart instance
/* ---------- Derived scores ----------
data/leaderboard.json holds only measured facts: how many problems each model
got right, what it spent to get there (tokens, or kWh for the ones we run
ourselves), and the price it is billed at. Everything below is computed here
on load: the percentages, the per-benchmark costs, the leetcode overall row
(problem-count-weighted), the knowledge/coding subscores, the composite, and
the totals. */
// Composite score weights. Every weight group sums to 1.
const KNOWLEDGE_SHARE = 0.5; // vs coding
const FREEFORM_WEIGHT = 0.7; // vs multiple choice (freeform is judged, no guessing floor)
const DIFFICULTY_WEIGHTS = { easy: 0.2, medium: 0.35, hard: 0.45 };
const CODING_METRIC_WEIGHTS = {
unitTestsPassed: 0.4,
fullySolved: 0.25,
partiallySolved: 0.15,
codeQuality: 0.1,
validSyntax: 0.1, // 100 - invalidSyntax
};
const DIFFICULTIES = ["easy", "medium", "hard"];
const QA_SPLITS = ["freeform", "multipleChoice"];
// How much of each split a topic view counts. Overall is the same 0.7/0.3 blend
// the knowledge subscore uses, so its bars end at that score; the single-split
// views weight one side fully and end at that split's own accuracy. Every view
// totals something already on the board, which is what makes the chart check
// itself.
const TOPIC_VIEWS = {
overall: {
label: "Knowledge",
weights: [FREEFORM_WEIGHT, 1 - FREEFORM_WEIGHT],
},
freeform: { label: "Freeform", weights: [1, 0] },
multipleChoice: { label: "Multiple choice", weights: [0, 1] },
};
const DIFF_METRICS = [
"unitTestsPassed",
"fullySolved",
"partiallySolved",
"codeQuality",
"invalidSyntax",
];
const slugify = (name) =>
name
.toLowerCase()
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-|-$/g, "");
const SPONSOR_URL = "https://github.com/sponsors/boatbomber";
const add = (values) => values.reduce((sum, v) => sum + v, 0);
const pct = (n, of) => (n / of) * 100;
// A missing piece makes the whole bill unknown rather than zero.
const addCosts = (costs) => (costs.some((c) => c == null) ? null : add(costs));
// Hosted models are billed by the token; the ones we run ourselves, by the meter.
function costOf(price, split) {
if (!price) return null;
if (price.electricity != null)
return split.kilowattHour == null
? null
: split.kilowattHour * price.electricity;
return (
(split.inputTokens * price.input + split.outputTokens * price.output) / 1e6
);
}
function codingScore(diff) {
const metrics = { ...diff, validSyntax: 100 - diff.invalidSyntax };
return Object.entries(CODING_METRIC_WEIGHTS).reduce(
(sum, [key, w]) => sum + metrics[key] * w,
0,
);
}
const MONTHS = [
"jan",
"feb",
"mar",
"apr",
"may",
"jun",
"jul",
"aug",
"sep",
"oct",
"nov",
"dec",
];
// releaseDate is an ISO day. The month-name form ("Mar 2026", "March 2025",
// "Sept 2025") is what the file used to hold and is still accepted, since the
// file is hand-maintained and a new entry may well be written that way — it
// lands mid-month, which is the honest position for a date known only to the
// month. Anything unrecognized comes back null and drops off the timeline
// rather than landing on the epoch.
function parseReleaseDate(text) {
const iso = /^(\d{4})-(\d{2})-(\d{2})$/.exec(text ?? "");
if (iso) return Date.UTC(+iso[1], +iso[2] - 1, +iso[3]);
const named = /^([A-Za-z]+)\s+(\d{4})$/.exec(text ?? "");
if (!named) return null;
const month = MONTHS.indexOf(named[1].slice(0, 3).toLowerCase());
return month < 0 ? null : Date.UTC(Number(named[2]), month, 15);
}
// Wilson score interval, returned as a half-width in percentage points. The
// textbook normal approximation collapses to zero width at 0% and 100%, which
// is where several of these proportions actually sit — Wilson's stays honest
// there, and at n=47 the difference is the whole point of showing an interval.
function wilsonHalfWidth(successes, n, z = 1.96) {
if (!n) return null;
const p = successes / n;
const denom = 1 + (z * z) / n;
const spread = z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n));
return (spread / denom) * 100;
}
function deriveScores(data) {
const qaBench = data.benchmarks.robloxqa;
const lcBench = data.benchmarks.leetcode;
const counts = lcBench.problemCounts;
const totalProblems = add(DIFFICULTIES.map((d) => counts[d]));
for (const m of data.models) {
m.id = slugify(m.name);
m.releaseTs = parseReleaseDate(m.releaseDate);
const s = m.scores;
const leet = s.leetcode;
// Correct counts -> percentages, plus each split's share of the bill. The
// raw numerator and denominator are kept alongside: a rate quoted without
// its sample size can't carry a confidence interval, and 47 problems and
// 3,000 questions do not deserve the same trust.
for (const d of DIFFICULTIES) {
const raw = leet[d];
leet[d] = {
...raw,
n: counts[d],
invalidSyntaxCount: raw.invalidSyntax,
// Code that parsed and still passed nothing. Invalid syntax is recorded
// as a subset of `fail` rather than beside it, so telling the two apart
// means taking one out of the other.
failedTests: raw.fail - raw.invalidSyntax,
unitTestsPassed: raw.unitTestPassRate,
fullySolved: pct(raw.fullPass, counts[d]),
// Partial passes are recorded exclusive of full ones; the metric is inclusive.
partiallySolved: pct(raw.fullPass + raw.partialPass, counts[d]),
invalidSyntax: pct(raw.invalidSyntax, counts[d]),
cost: costOf(m.price, raw),
};
}
leet.cost = addCosts(DIFFICULTIES.map((d) => leet[d].cost));
// Each QA split is scored and priced on its own questions and tokens, so it
// gets its own row shape; the overall row compares the two.
const quiz = s.robloxqa;
const qaCounts = qaBench.problemCounts;
for (const k of QA_SPLITS) {
const raw = quiz[k];
quiz[k] = {
...raw,
n: qaCounts[k],
correctCount: raw.correct,
noAnswerCount: raw.noAnswer,
correct: pct(raw.correct, qaCounts[k]),
noAnswer: pct(raw.noAnswer, qaCounts[k]),
cost: costOf(m.price, raw),
};
}
const pooled = (key) => add(QA_SPLITS.map((k) => quiz[k][key]));
quiz.overall = {
freeform: quiz.freeform.correct,
multipleChoice: quiz.multipleChoice.correct,
// The splits ask the same number of questions, so a plain mean of their
// rates is the pooled rate.
noAnswer: add(QA_SPLITS.map((k) => quiz[k].noAnswer)) / QA_SPLITS.length,
// ...and the pooled counts behind it, for the interval on that rate.
n: pooled("n"),
correctCount: pooled("correctCount"),
noAnswerCount: pooled("noAnswerCount"),
outputTokens: pooled("outputTokens"),
cost: addCosts(QA_SPLITS.map((k) => quiz[k].cost)),
};
// Leetcode overall: metrics weighted by problem count, tokens summed.
const overall = {};
for (const key of DIFF_METRICS) {
overall[key] =
DIFFICULTIES.reduce((sum, d) => sum + leet[d][key] * counts[d], 0) /
totalProblems;
}
// Weighting each difficulty's rate by its problem count is the pooled rate,
// so the summed counts are the numerators those percentages came from.
overall.n = totalProblems;
for (const key of [
"fullPass",
"partialPass",
"failedTests",
"invalidSyntaxCount",
"noCode",
])
overall[key] = add(DIFFICULTIES.map((d) => leet[d][key]));
overall.outputTokens = add(DIFFICULTIES.map((d) => leet[d].outputTokens));
overall.cost = leet.cost;
leet.overall = overall;
s.knowledge =
FREEFORM_WEIGHT * quiz.overall.freeform +
(1 - FREEFORM_WEIGHT) * quiz.overall.multipleChoice;
s.coding = DIFFICULTIES.reduce(
(sum, d) => sum + codingScore(leet[d]) * DIFFICULTY_WEIGHTS[d],
0,
);
s.overall =
KNOWLEDGE_SHARE * s.knowledge + (1 - KNOWLEDGE_SHARE) * s.coding;
// The same five products the composite is built from, kept as a list so the
// decomposition chart plots the score rather than a restatement of it. They
// sum to s.overall by construction, which is what makes that chart worth
// trusting.
s.contributions = [
{
label: "Knowledge · Freeform",
points: KNOWLEDGE_SHARE * FREEFORM_WEIGHT * quiz.overall.freeform,
from: quiz.overall.freeform,
},
{
label: "Knowledge · Multiple choice",
points:
KNOWLEDGE_SHARE * (1 - FREEFORM_WEIGHT) * quiz.overall.multipleChoice,
from: quiz.overall.multipleChoice,
},
...DIFFICULTIES.map((d) => ({
label: `Coding · ${d[0].toUpperCase()}${d.slice(1)}`,
points:
(1 - KNOWLEDGE_SHARE) * DIFFICULTY_WEIGHTS[d] * codingScore(leet[d]),
from: codingScore(leet[d]),
})),
];
// The knowledge half again, cut by the area of the docs each question was
// grounded in. Both splits ask about the same pages, so a bucket has one
// denominator in each and the 0.7/0.3 blend that makes s.knowledge applies
// bucket by bucket — which is why these sum to it rather than merely
// resembling it. A bucket's contribution is therefore its own accuracy
// scaled by its share of the 3,000 questions, and the big buckets carry
// more of the bar for the same reason they carry more of the score.
const topicDefs = qaBench.topics ?? [];
const perSplit = add(topicDefs.map((t) => t.n));
s.topicContributions = Object.fromEntries(
Object.entries(TOPIC_VIEWS).map(([view, { weights }]) => [
view,
topicDefs.map((t) => {
const counts = quiz.topics?.[t.key] ?? {};
const weighted = add(
QA_SPLITS.map((k, i) => weights[i] * (counts[k] ?? 0)),
);
return {
label: t.label,
n: t.n,
points: pct(weighted, perSplit),
from: pct(weighted, t.n),
};
}),
]),
);
s.totalCost = addCosts([quiz.overall.cost, leet.cost]);
// What a point of composite score cost to buy. Ranks value directly, where
// the cost column alone rewards a model for being bad cheaply.
s.costPerPoint =
s.totalCost == null || !s.overall ? null : s.totalCost / s.overall;
s.totalOutputTokens = quiz.overall.outputTokens + overall.outputTokens;
}
data.models.sort((a, b) => b.scores.overall - a.scores.overall);
}
function totalEvalCost(data) {
return data.models.reduce((sum, m) => sum + m.scores.totalCost, 0);
}
function easeOutCubic(t) {
return 1 - (1 - t) ** 3;
}
const clamp = (v, lo, hi) => Math.min(Math.max(v, lo), hi);
function smootherstep(x) {
return x * x * x * (x * (x * 6 - 15) + 10);
}
function renderEvalCostTotal() {
const el = document.getElementById("eval-cost-total");
if (!el) return;
// Round to the cent we actually display: costs are derived at full precision,
// and the fastest wheel spins continuously, so a target of 382.664 parks it
// 40% of the way past the 6 instead of on it.
const total = Math.round(totalEvalCost(state.data) * 100) / 100;
const start = total - 175;
// Build a spinner odometer: a column per digit of "$XX.XX", with the static
// "$" and "." rendered between them. Each column stacks digits 0-9 (plus a
// trailing 0 so the 9->0 wrap slides seamlessly) and is translated vertically.
const [intPart, decPart] = total.toFixed(2).split(".");
const strip = `${Array.from(
{ length: 11 },
(_, i) => `${i % 10}`,
).join("")}`;
let markup = '$';
for (let i = 0; i < intPart.length; i++) {
const place = 10 ** (intPart.length - 1 - i);
markup += `${strip}`;
}
markup += '.';
for (let i = 0; i < decPart.length; i++) {
markup += `${strip}`;
}
el.innerHTML = `Total eval cost: ${markup} · Sponsor on GitHub ↗`;
el.hidden = false;
const valueEl = el.querySelector(".eval-cost-value");
if (!valueEl) return;
const cols = [...valueEl.querySelectorAll(".odo-col")].map((c) => ({
el: c,
strip: c.querySelector(".odo-strip"),
place: parseFloat(c.dataset.place),
isInt: c.dataset.int === "1",
}));
// A wheel at place p spins at (value velocity / p), so smaller places spin
// faster. Precompute each column's relative spin speed on a log scale (the
// places are powers of ten, so this spreads evenly across the digits): the
// fastest wheel is 1 and the slowest is 0. This drives the per-digit tint.
const minPlace = cols[cols.length - 1].place;
const decades = Math.log10(cols[0].place / minPlace);
for (const col of cols) {
col.speed =
decades > 0
? clamp(1 + Math.log10(minPlace / col.place) / decades, 0, 1)
: 1;
}
// Position every column for a given value, odometer-style: the fastest wheel
// (smallest place, last column) spins continuously, while each higher wheel
// rests exactly on its digit and only rolls over as the wheel just below it
// sweeps from 9 back to 0. Leading integer zeros are hidden.
// Walk bottom-up: a wheel carries off the offset actually rendered below it,
// not that wheel's raw value. Only the fastest wheel sits between digits, so
// reading the raw value would roll every wheel above a resting 9 partway over.
function place(value) {
let below = null;
for (let i = cols.length - 1; i >= 0; i--) {
const col = cols[i];
let offset;
if (i === cols.length - 1) {
offset = (value / col.place) % 10; // continuous position in [0, 10)
} else {
const digit = Math.floor(value / col.place) % 10;
const carry = smootherstep(clamp(below - 9, 0, 1)); // rolls only across 9 -> 0
offset = digit + carry;
}
col.strip.style.transform = `translateY(${-offset}em)`;
if (col.isInt) col.el.style.opacity = value < col.place ? "0" : "1";
below = offset;
}
}
if (window.matchMedia("(prefers-reduced-motion: reduce)").matches) {
place(total);
return;
}
const duration = 5000;
let startTime = null;
function tick(now) {
if (!startTime) startTime = now;
const t = Math.min((now - startTime) / duration, 1);
place(start + (total - start) * easeOutCubic(t));
// Tint each digit toward the brand purple based on how fast its wheel is
// spinning: the eased value slows to a stop (normalized velocity, the
// derivative of easeOutCubic, runs 1 -> 0), and within any instant faster
// wheels (the cents) run purpler than slower ones (the hundreds).
const velocity = (1 - t) ** 2;
for (const col of cols) {
const tint = (velocity * col.speed) ** 0.7 * 100;
col.el.style.color = `color-mix(in srgb, var(--color-primary-bright) ${tint}%, var(--color-text-secondary))`;
}
if (t < 1) requestAnimationFrame(tick);
else {
place(total);
for (const col of cols) col.el.style.color = "";
}
}
requestAnimationFrame(tick);
}
/* ---------- Value accessors & formatting ---------- */
const lc = (m, diff) => m.scores.leetcode[diff ?? state.difficulty];
const qa = (m, split) => m.scores.robloxqa[split ?? state.qaSplit];
const fmtPct = (v) => (v == null ? "—" : v.toFixed(1));
const fmtInt = (v) => (v == null ? "—" : Math.round(v).toString());
const fmtTokens = (v) => (v == null ? "—" : v.toLocaleString("en-US"));
const fmtCost = (v) => (v == null ? "—" : `$${v.toFixed(2)}`);
// A point of score costs cents at the top of the board and thousandths at the
// bottom, so two decimals would round most of the column to $0.00.
const fmtFineCost = (v) => (v == null ? "—" : `$${v.toFixed(v < 0.1 ? 3 : 2)}`);
const MONTH_LABELS = MONTHS.map((m) => m[0].toUpperCase() + m.slice(1));
const fmtMonth = (ts) => {
const d = new Date(ts);
return `${MONTH_LABELS[d.getUTCMonth()]} ${d.getUTCFullYear()}`;
};
// Axis ticks stay at month granularity; anywhere a single model is named, the
// day it shipped is worth the four extra characters.
const fmtDay = (ts) => {
const d = new Date(ts);
return `${MONTH_LABELS[d.getUTCMonth()]} ${d.getUTCDate()}, ${d.getUTCFullYear()}`;
};
function fmtBytes(v) {
if (v == null) return "—";
const units = ["B", "KB", "MB", "GB", "TB"];
let i = 0;
while (v >= 1024 && i < units.length - 1) {
v /= 1024;
i++;
}
return `${v >= 10 ? Math.round(v) : v.toFixed(1)} ${units[i]}`;
}
// How each column type renders when it doesn't bring its own `fmt`.
const TYPE_FORMATS = {
score: fmtPct,
tokens: fmtTokens,
cost: fmtCost,
plain: fmtPct,
};
// type: "score" gets a purple background tinted by value (values are 0-100)
const COLUMN_SETS = {
overview: [
{
key: "overall",
label: "Score",
type: "score",
get: (m) => m.scores.overall,
},
{
key: "knowledge",
label: "Knowledge",
type: "score",
get: (m) => m.scores.knowledge,
},
{
key: "coding",
label: "Coding",
type: "score",
get: (m) => m.scores.coding,
},
{
key: "tokens",
label: "Out Tokens",
type: "tokens",
lowerBetter: true,
get: (m) => m.scores.totalOutputTokens,
},
{
key: "cost",
label: "Cost",
type: "cost",
lowerBetter: true,
get: (m) => m.scores.totalCost,
},
{
key: "costPerPoint",
label: "Cost / Point",
type: "cost",
lowerBetter: true,
get: (m) => m.scores.costPerPoint,
fmt: fmtFineCost,
},
],
robloxqa: [
{
key: "freeform",
label: "Freeform %",
type: "score",
get: (m) => qa(m, "overall").freeform,
ci: (m) => ({
k: qa(m, "freeform").correctCount,
n: qa(m, "freeform").n,
}),
},
{
key: "mc",
label: "Multi Choice %",
type: "score",
get: (m) => qa(m, "overall").multipleChoice,
ci: (m) => ({
k: qa(m, "multipleChoice").correctCount,
n: qa(m, "multipleChoice").n,
}),
},
{
key: "noAnswer",
label: "No Answer %",
type: "plain",
lowerBetter: true,
get: (m) => qa(m).noAnswer,
ci: (m) => ({ k: qa(m).noAnswerCount, n: qa(m).n }),
},
{
key: "tokens",
label: "Out Tokens",
type: "tokens",
lowerBetter: true,
get: (m) => qa(m).outputTokens,
},
{
key: "cost",
label: "Cost",
type: "cost",
lowerBetter: true,
get: (m) => qa(m).cost,
},
],
// One split on its own: the two percentages that split the questions between
// them are answered-right and never-answered.
robloxqaSplit: [
{
key: "correct",
label: "Correct %",
type: "score",
get: (m) => qa(m).correct,
ci: (m) => ({ k: qa(m).correctCount, n: qa(m).n }),
},
{
key: "noAnswer",
label: "No Answer %",
type: "plain",
lowerBetter: true,
get: (m) => qa(m).noAnswer,
ci: (m) => ({ k: qa(m).noAnswerCount, n: qa(m).n }),
},
{
key: "tokens",
label: "Out Tokens",
type: "tokens",
lowerBetter: true,
get: (m) => qa(m).outputTokens,
},
{
key: "cost",
label: "Cost",
type: "cost",
lowerBetter: true,
get: (m) => qa(m).cost,
},
],
leetcode: [
{
key: "tests",
label: "Unit Tests %",
type: "score",
get: (m) => lc(m).unitTestsPassed,
},
{
key: "solved",
label: "Fully Solved %",
type: "score",
get: (m) => lc(m).fullySolved,
ci: (m) => ({ k: lc(m).fullPass, n: lc(m).n }),
},
{
key: "partial",
label: "Partial %",
type: "score",
get: (m) => lc(m).partiallySolved,
// The metric is inclusive of full passes; the count has to match it.
ci: (m) => ({ k: lc(m).fullPass + lc(m).partialPass, n: lc(m).n }),
},
{
key: "quality",
label: "Quality",
type: "score",
get: (m) => lc(m).codeQuality,
fmt: fmtInt,
},
{
key: "syntax",
label: "Bad Syntax %",
type: "plain",
lowerBetter: true,
get: (m) => lc(m).invalidSyntax,
ci: (m) => ({ k: lc(m).invalidSyntaxCount, n: lc(m).n }),
},
{
key: "tokens",
label: "Out Tokens",
type: "tokens",
lowerBetter: true,
get: (m) => lc(m).outputTokens,
},
{
key: "cost",
label: "Cost",
type: "cost",
lowerBetter: true,
get: (m) => lc(m).cost,
},
],
};
function activeColumns() {
// A single QA split swaps the whole column set; leetcode keeps its columns
// across difficulties and only changes which rows they read.
if (state.columnSet === "robloxqa" && state.qaSplit !== "overall")
return COLUMN_SETS.robloxqaSplit;
return COLUMN_SETS[state.columnSet];
}
/* ---------- Filtering ---------- */
// The one gate every view passes through, so the search box and the weights
// filter reach the charts and the table alike without either knowing about them.
function filteredModels() {
const q = state.search.trim().toLowerCase();
return state.data.models.filter(
(m) =>
// Published weights are the ones with a size on disk to publish.
(state.weightsFilter !== "open" || m.size != null) &&
(!q || `${m.name} ${m.provider}`.toLowerCase().includes(q)),
);
}
function compareBy(get, a, b, dir) {
const va = get(a);
const vb = get(b);
if (va == null && vb == null) return 0;
if (va == null) return 1; // nulls last regardless of direction
if (vb == null) return -1;
return (va - vb) * dir;
}
function sortedModels(models) {
const cols = activeColumns();
const idx = cols.findIndex((c) => c.key === state.sortKey);
const primary = idx >= 0 ? cols[idx] : null;
const get = primary ? primary.get : (m) => m.scores.overall;
// Tie-breakers: remaining columns in order, starting after the sorted one,
// each compared in its natural direction (lower is better for cost-like columns)
const tiebreakers = primary
? cols.slice(idx + 1).concat(cols.slice(0, idx))
: cols;
return [...models].sort((a, b) => {
let cmp = compareBy(get, a, b, state.sortDir);
for (const c of tiebreakers) {
if (cmp !== 0) break;
cmp = compareBy(c.get, a, b, c.lowerBetter ? 1 : -1);
}
return cmp;
});
}
/* ---------- Table rendering ---------- */
function providerColor(p) {
if (p === "Torpedo Software") return cssVar("--color-primary-bright");
return PROVIDER_COLORS[p] ?? cssVar("--color-text-tertiary");
}
function relativeLuminance(color) {
const hex = /^#([0-9a-f]{6})$/i.exec(color);
if (!hex) return 0; // unknown format: treat as dark, i.e. use the white glyph
const [r, g, b] = [0, 2, 4]
.map((i) => parseInt(hex[1].slice(i, i + 2), 16) / 255)
.map((c) => (c <= 0.04045 ? c / 12.92 : ((c + 0.055) / 1.055) ** 2.4));
return 0.2126 * r + 0.7152 * g + 0.0722 * b;
}
// The glyphs ship white, which disappears on the paler brand colors.
const glyphNeedsDarkInk = (color) => relativeLuminance(color) > 0.45;
function providerInk(color) {
return glyphNeedsDarkInk(color) ? "#12141a" : "#ffffff";
}
// The badge beside a model's name: the same disc-and-glyph the chart plots, but
// assembled from an
so it stays sharp at any size and needs nothing
// preloaded. Providers without a logo file keep the plain colored dot.
function providerMarkHtml(provider) {
const color = providerColor(provider);
const logo = PROVIDER_LOGOS[provider];
if (!logo) return ``;
// A full-bleed logo is its own finished badge, so it needs no disc under it.
const cls = FULL_BLEED_LOGOS.has(provider)
? "provider-mark is-badge"
: `provider-mark${glyphNeedsDarkInk(color) ? " ink-dark" : ""}`;
const disc = FULL_BLEED_LOGOS.has(provider)
? ""
: ` style="background:${color}"`;
return `
`;
}
function renderTable() {
const cols = activeColumns();
const thead = document.querySelector("#board-table thead");
const tbody = document.querySelector("#board-table tbody");
const headCells = [
`
# | `,
`Model | `,
...cols.map((c) => {
const sorted = c.key === state.sortKey;
const arrow = sorted ? (state.sortDir < 0 ? "▾" : "▴") : "";
// Headers wrap to a second line when the table is tight, so glue the unit
// and the sort arrow to the word before them rather than let either orphan.
const label = c.label.replace(/ ([%$])$/, "\u00a0$1");
return `${label} ${arrow} | `;
}),
];
thead.innerHTML = `${headCells.join("")}
`;
const models = sortedModels(filteredModels());
tbody.innerHTML = models.length
? models.map((m, i) => modelRowHtml(m, i + 1, cols)).join("")
: `No models match your search. |
`;
// Column count changes with the view, so the table's width does too.
updateScrollShadows();
}
// The 95% margin a percentage carries, for the columns that named the counts
// they were computed from. A column that names none gets nothing: unit tests
// passed and code quality are means of per-problem values rather than
// proportions, and the composite scores are weighted blends, so a binomial
// interval on any of them would be invented.
function marginHtml(col, m) {
if (!col.ci) return "";
const { k, n } = col.ci(m);
const half = wilsonHalfWidth(k, n);
return half == null ? "" : ` ±${half.toFixed(1)}`;
}
function modelRowHtml(m, rank, cols) {
const medal = rank <= 3 ? `${rank}` : rank;
const cells = cols
.map((c) => {
const v = c.get(m);
const text =
(c.fmt ?? TYPE_FORMATS[c.type] ?? fmtPct)(v) + marginHtml(c, m);
if (c.type !== "score") return `${text} | `;
const share = v == null ? 0 : Math.max(0, Math.min(1, v / 100));
// Power curve so the tint separates top scores more than a
// linear ramp would: 90 stays vivid while 60 fades well back.
const tint = Math.pow(share, 2.5);
return `${text} | `;
})
.join("");
const expanded = state.expandedId === m.id;
let rows = `
| ${medal} |
${m.name}
${providerMarkHtml(m.provider)}${m.provider}${m.releaseTs ? ` · ${fmtDay(m.releaseTs)}` : ""}
| ${cells}
`;
if (expanded) rows += detailRowHtml(m, cols.length + 2);
return rows;
}
// Shade an edge only while there are columns hidden behind it, so the fade is a
// scroll affordance rather than permanent decoration.
function updateScrollShadows() {
const scroller = document.querySelector(".table-scroll");
const frame = scroller.closest(".table-frame");
const hidden = scroller.scrollWidth - scroller.clientWidth;
frame.classList.toggle("can-scroll-left", scroller.scrollLeft > 1);
frame.classList.toggle("can-scroll-right", scroller.scrollLeft < hidden - 1);
}
function detailRowHtml(m, colspan) {
const diffTable = `
| Tests | Solved | Partial | Quality | Bad syntax | Tokens | Cost |
${DIFFICULTIES.map((d) => {
const s = m.scores.leetcode[d];
return `| ${d} | ${fmtPct(s.unitTestsPassed)}% | ${fmtPct(s.fullySolved)}% | ${fmtPct(s.partiallySolved)}% | ${fmtInt(s.codeQuality)} | ${fmtPct(s.invalidSyntax)}% | ${fmtTokens(s.outputTokens)} | ${fmtCost(s.cost)} |
`;
}).join("")}
`;
const QA_LABELS = { freeform: "freeform", multipleChoice: "multiple choice" };
const splitTable = `
| Correct | No answer | Tokens | Cost |
${QA_SPLITS.map((k) => {
const s = qa(m, k);
return `| ${QA_LABELS[k]} | ${fmtPct(s.correct)}% | ${fmtPct(s.noAnswer)}% | ${fmtTokens(s.outputTokens)} | ${fmtCost(s.cost)} |
`;
}).join("")}
`;
return `
RobloxQA by split${splitTable}
Luau Leetcode by difficulty${diffTable}
|
`;
}
/* ---------- Charts ---------- */
function chartDefaults() {
// Nothing animates. A zoom or pan moves the scales at once while animated
// elements glide toward the new range, so for the length of the transition
// the points sit where their own axes say they don't belong. Being right at
// every frame beats easing into being right.
Chart.defaults.animation = false;
Chart.defaults.color = cssVar("--color-text-tertiary");
Chart.defaults.borderColor = cssVar("--color-border-subtle");
Chart.defaults.font.family = "'Inter', sans-serif";
Chart.defaults.font.size = 12;
Chart.defaults.devicePixelRatio = renderScale();
}
// Axis titles carry their units in parentheses ("Total cost (USD, log)"), which
// is noise in a tooltip that already formats the value beside it.
const plainTitle = (title) => title.replace(/\s*\(.*\)$/, "");
// Axis ticks have to stay short: "1.2M" reads at a glance where "1,200,000"
// forces the labels to rotate.
const fmtCompact = (v) =>
v >= 1e6
? `${+(v / 1e6).toFixed(1)}M`
: v >= 1e3
? `${Math.round(v / 1e3)}K`
: `${v}`;
// The size axis names every power-of-two step it draws a line for, so its
// labels have to be short: "128G" where the table would write "128 GB".
const fmtBytesTick = (v) => {
const [num, unit] = fmtBytes(v).split(" ");
return num.replace(".0", "") + unit[0];
};
const gridColor = () => withAlpha(cssVar("--color-border"), 0.55);
/* ---------- Provider logo markers ---------- */
function loadImage(src) {
return new Promise((resolve) => {
const img = new Image();
img.addEventListener("load", () => resolve(img));
img.addEventListener("error", () => resolve(null)); // a missing logo just falls back to a plain disc
img.src = src;
});
}
async function loadProviderLogos() {
const loaded = await Promise.all(
Object.entries(PROVIDER_LOGOS).map(async ([provider, src]) => [
provider,
await loadImage(src),
]),
);
for (const [provider, img] of loaded) if (img) logoImages[provider] = img;
}
// Canvas has no CSS mask, so paint the loaded shape through itself: draw it,
// then flood the box with the ink color clipped to the pixels already there.
function tintedGlyph(logo, color, box) {
const scale = Math.min(box / logo.naturalWidth, box / logo.naturalHeight);
const canvas = document.createElement("canvas");
canvas.width = Math.max(1, Math.round(logo.naturalWidth * scale));
canvas.height = Math.max(1, Math.round(logo.naturalHeight * scale));
const ctx = canvas.getContext("2d");
ctx.drawImage(logo, 0, 0, canvas.width, canvas.height);
ctx.globalCompositeOperation = "source-in";
ctx.fillStyle = color;
ctx.fillRect(0, 0, canvas.width, canvas.height);
return canvas;
}
// One marker bitmap, drawn supersampled so it stays sharp on high-DPI screens.
function drawMarker(provider, size) {
const supersample = markerScale();
const px = size * supersample;
const canvas = document.createElement("canvas");
canvas.width = px;
canvas.height = px;
const ctx = canvas.getContext("2d");
const logo = logoImages[provider];
if (logo && FULL_BLEED_LOGOS.has(provider)) {
ctx.drawImage(logo, 0, 0, px, px);
return canvas;
}
const color = providerColor(provider);
const ring = MARKER_RING * supersample;
ctx.beginPath();
ctx.arc(px / 2, px / 2, (px - ring) / 2, 0, Math.PI * 2);
ctx.fillStyle = color;
ctx.fill();
ctx.lineWidth = ring;
ctx.strokeStyle = withAlpha(cssVar("--color-canvas"), 0.9);
ctx.stroke();
if (logo) {
const glyph = tintedGlyph(logo, providerInk(color), px * GLYPH_INSET);
ctx.drawImage(glyph, (px - glyph.width) / 2, (px - glyph.height) / 2);
}
return canvas;
}
// Chart.js draws an image point at img.width/height rather than at the point
// radius, so the supersampled bitmap has to come back as an
sized down to
// the display diameter — a canvas would draw at its own bitmap size. Colors are
// read from the theme, so this reruns whenever the theme flips.
async function buildProviderMarkers() {
await Promise.all(
Object.keys(PROVIDER_LOGOS).flatMap((provider) =>
[MARKER_SIZE, MARKER_HOVER_SIZE].map(async (size) => {
const img = await loadImage(drawMarker(provider, size).toDataURL());
if (!img) return;
img.width = size;
img.height = size;
markerImages[`${provider}|${size}`] = img;
}),
),
);
}
function providerMarker(provider, active) {
const size = active ? MARKER_HOVER_SIZE : MARKER_SIZE;
return markerImages[`${provider}|${size}`] ?? "circle";
}
function scatterDataset(models, getX, getY) {
// Markers overlap wherever the field bunches up, and Chart.js draws in data
// order. Plotting the weakest first leaves the models people came to read on
// top instead of half-buried.
const points = models
.map((m) => ({ x: getX(m), y: getY(m), model: m }))
.sort((a, b) => a.y - b.y);
return {
data: points,
// The marker bakes in its own disc and ring; these are what a point falls
// back to before the markers finish loading.
backgroundColor: points.map((p) => providerColor(p.model.provider)),
borderColor: withAlpha(cssVar("--color-canvas"), 0.9),
borderWidth: 1.5,
pointStyle: (ctx) =>
providerMarker(points[ctx.dataIndex]?.model.provider, ctx.active),
pointRadius: MARKER_SIZE / 2, // hit testing still goes by radius
pointHoverRadius: MARKER_HOVER_SIZE / 2,
};
}
/* ---------- Zoom & pan ---------- */
// The plugin ships from a CDN and the board has to survive without it.
const zoomAvailable = () => Boolean(Chart.registry.plugins.get("zoom"));
// A drag that happens to end over a point shouldn't also open that model's page.
let panSuppressesClick = false;
function syncZoomState(chart) {
chart.canvas.parentElement.classList.toggle(
"is-zoomed",
chart.isZoomedOrPanned(),
);
}
// Zoom belongs to the scatter plots, where the models worth comparing pile into
// one corner; the difficulty chart is a ranking, not a field to explore. The
// wheel takes a modifier because these sit in a long scrolling page and
// swallowing the wheel would trap anyone scrolling past. Touch gestures need
// hammerjs, which isn't loaded, so they never fire — which is what leaves
// one-finger page scrolling over a chart alone on phones.
function zoomPluginOptions() {
if (!zoomAvailable()) return undefined;
return {
// Panning stops at the edges of the full view: there is nothing out there.
limits: {
x: { min: "original", max: "original" },
y: { min: "original", max: "original" },
},
pan: {
enabled: true,
mode: "xy",
threshold: 6, // px of travel before a click becomes a drag
onPanStart: () => {
panSuppressesClick = false;
},
onPan: ({ chart }) => {
// At full extent the limits clamp a pan to nothing, so the gesture was
// a page scroll or an idle drag, and the click ending it is a real one.
panSuppressesClick = chart.isZoomedOrPanned();
},
onPanComplete: ({ chart }) => syncZoomState(chart),
},
zoom: {
mode: "xy",
wheel: { enabled: true, modifierKey: "ctrl", speed: 0.075 },
pinch: { enabled: false }, // handled in attachPinch, see why there
onZoomComplete: ({ chart }) => syncZoomState(chart),
},
};
}
// Chart.js sizes an axis box to fit whatever its tick labels currently
// measure, so a zoom that turns "100" into "81.5" widens the y axis, walks the
// plot area's left edge inward, and drags every point along with it — for
// reasons that have nothing to do with the data. Freeze both boxes at the size
// the full view settled on: after that, zooming moves the points and nothing
// else. Only the reset view gets a vote on the layout.
const axisBoxes = new WeakMap(); // chart -> axis id -> frozen geometry
// An axis reserves its own thickness plus enough padding along it for the first
// and last labels to overhang, and both move as the labels change. Together
// they are every edge of the plot area.
const AXIS_BOX_PROPS = {
x: ["height", "paddingLeft", "paddingRight"],
y: ["width", "paddingTop", "paddingBottom"],
};
function freezeAxisBoxes(chart) {
const boxes = {};
for (const [id, props] of Object.entries(AXIS_BOX_PROPS)) {
const axis = chart.scales[id];
if (axis) boxes[id] = Object.fromEntries(props.map((p) => [p, axis[p]]));
}
axisBoxes.set(chart, boxes);
}
function applyFrozenBox(axis) {
const frozen = axisBoxes.get(axis.chart)?.[axis.axis];
if (frozen) Object.assign(axis, frozen);
}
// A responsive resize is the one time the boxes should be re-measured: a
// narrower card genuinely needs a different axis than a wide one.
function remeasureAxisBoxes(chart) {
axisBoxes.delete(chart);
requestAnimationFrame(() => freezeAxisBoxes(chart));
}
// Pinch-to-zoom, by hand. The plugin's own pinch reads the angle between your
// two fingers and scales only the axes it infers from it: side by side zooms x,
// stacked zooms y, and only a roughly diagonal pinch gets both. The axis
// it skips stays pinned at full extent, so afterwards that direction won't pan
// either. A scatter plot isn't a photograph — a pinch zooms both axes, always.
function attachPinch(chart) {
const canvas = chart.canvas;
const fingers = new Map(); // pointerId -> client position
let spread = null; // distance between the two fingers, as of the last frame
const pair = () => [...fingers.values()].slice(0, 2);
const spreadOf = ([a, b]) => Math.hypot(a.x - b.x, a.y - b.y);
const focalOf = ([a, b]) => {
const rect = canvas.getBoundingClientRect();
return { x: (a.x + b.x) / 2 - rect.left, y: (a.y + b.y) / 2 - rect.top };
};
const resync = () => {
spread = fingers.size === 2 ? spreadOf(pair()) : null;
};
// Assigned rather than added: renderCharts rebuilds the chart on the same
// canvas, and listeners would pile up one deep per rebuild.
canvas.onpointerdown = (e) => {
if (e.pointerType === "mouse") return; // the wheel covers the mouse
fingers.set(e.pointerId, { x: e.clientX, y: e.clientY });
resync();
};
canvas.onpointermove = (e) => {
if (!fingers.has(e.pointerId)) return;
fingers.set(e.pointerId, { x: e.clientX, y: e.clientY });
if (fingers.size !== 2) return; // one finger pans, three are nothing
const next = spreadOf(pair());
if (spread > 0 && next > 0) {
const rate = next / spread;
chart.zoom({ x: rate, y: rate, focalPoint: focalOf(pair()) });
syncZoomState(chart);
}
spread = next;
};
const lift = (e) => {
fingers.delete(e.pointerId);
resync();
};
canvas.onpointerup = lift;
canvas.onpointercancel = lift;
}
// Built here rather than in the markup so the control only ever exists on a
// chart that can actually be reset.
function attachZoomControls(chart) {
const box = chart.canvas.parentElement;
const reset =
box.querySelector(".chart-reset") ??
box.appendChild(document.createElement("button"));
reset.type = "button";
reset.className = "chart-reset";
reset.textContent = "Reset view";
const restore = () => {
chart.resetZoom();
syncZoomState(chart);
};
reset.onclick = restore;
chart.canvas.ondblclick = restore;
attachPinch(chart);
freezeAxisBoxes(chart); // the chart has had its first layout by now
syncZoomState(chart); // a rebuilt chart is back at its full range
}
function scatterOptions(xTitle, yTitle, logX, xFormat, xTick) {
// includeBounds would pin a tick to each end of the range. At full extent
// those are round numbers; zoomed, they are wherever the wheel stopped, and
// an axis reading "$20.891172765721926" next to "$23" is noise.
const ticks = { maxTicksLimit: 8, maxRotation: 45, includeBounds: false };
if (xTick) {
// The scale decides which ticks earn a label and blanks the rest (a log
// axis labels only its majors); this restyles the ones it kept.
const format = logX
? Chart.Ticks.formatters.logarithmic
: Chart.Ticks.formatters.numeric;
ticks.callback = function (value, index, all) {
return format.call(this, value, index, all) === "" ? "" : xTick(value);
};
}
return {
responsive: true,
maintainAspectRatio: false,
onResize: remeasureAxisBoxes,
onClick(evt, elements, chart) {
if (panSuppressesClick) {
panSuppressesClick = false; // the tail of a drag, not a click
return;
}
const el = elements.find(
(e) => chart.data.datasets[e.datasetIndex].data[e.index]?.model,
);
const m = el && chart.data.datasets[el.datasetIndex].data[el.index].model;
if (m) window.open(m.url, "_blank", "noopener");
},
onHover(evt, elements, chart) {
const canvas = evt.native.target;
// The grab cursor only shows once there is somewhere to pan to: at full
// extent the limits pin the view, so offering to drag it would be a lie.
const pannable = zoomAvailable() && chart.isZoomedOrPanned();
if (pannable && evt.native.buttons) canvas.style.cursor = "grabbing";
else if (elements.length) canvas.style.cursor = "pointer";
else canvas.style.cursor = pannable ? "grab" : "default";
},
plugins: {
legend: { display: false },
zoom: zoomPluginOptions(),
tooltip: {
// The point's own marker becomes the swatch, so the provider stays
// readable in the tooltip too — pinned to the resting size, since the
// point it describes is by definition the hovered (larger) one.
usePointStyle: true,
boxWidth: MARKER_SIZE,
boxHeight: MARKER_SIZE,
boxPadding: 6,
padding: 10,
titleFont: { size: 13 },
bodyFont: { size: 12 },
bodySpacing: 4,
filter: (item) => Boolean(item.raw?.model), // the reference lines carry no model
callbacks: {
title: (items) => items[0].raw.model.name,
labelPointStyle: (ctx) => ({
pointStyle: providerMarker(ctx.raw.model.provider, false),
rotation: 0,
}),
label: (ctx) => [
ctx.raw.model.provider,
`${plainTitle(yTitle)}: ${ctx.parsed.y.toFixed(1)}`,
`${plainTitle(xTitle)}: ${xFormat(ctx.parsed.x)}`,
],
},
},
},
scales: {
x: {
type: logX ? "logarithmic" : "linear",
title: { display: true, text: xTitle },
grid: { color: gridColor() },
ticks,
afterFit: applyFrozenBox,
},
y: {
title: { display: true, text: yTitle },
grid: { color: gridColor() },
ticks: { includeBounds: false },
afterFit: applyFrozenBox,
},
},
};
}
const DAY_MS = 24 * 3600 * 1000;
const MONTH_MS = 30.44 * DAY_MS;
// Month starts inside the range, for the release axis: every third one across
// the whole field, every one once zoomed in. Counting months from year zero
// keeps the walk to integer math, and flooring to a multiple of the step keeps
// the ladder from shifting under a pan. The scale's own ticks are no fallback
// here the way they are on the size axis — numeric tick values inside a single
// month all format to the same label, so the axis would read "Apr 2026" five
// times. RELEASE_MIN_RANGE is what stops it getting that far.
function releaseTicks(min, max) {
const step = max - min > 10 * MONTH_MS ? 3 : 1;
const at = (i) => Date.UTC(Math.floor(i / 12), i % 12, 1);
const from = new Date(min);
let i = from.getUTCFullYear() * 12 + from.getUTCMonth();
i -= i % step;
while (at(i) < min) i += step;
const ticks = [];
for (; at(i) <= max && ticks.length < 24; i += step) ticks.push(at(i));
return ticks;
}
// Release dates carry one month of resolution, so a quarter is as far in as
// there is anything to see.
const RELEASE_MIN_RANGE = 3 * MONTH_MS;
// The variants that make a bar its own row rather than a point in a field.
const shortName = (m) => m.name.replace(/\s*\(.*\)$/, "");
// Three charts now rank every model as a horizontal row. None is a field worth
// panning around, so none of them takes the zoom plugin — see zoomPluginOptions.
function rowChartOptions({ stacked = false, callbacks = {} } = {}) {
return {
indexAxis: "y",
responsive: true,
maintainAspectRatio: false,
plugins: {
legend: { position: "top", labels: { boxWidth: 12, boxHeight: 12 } },
tooltip: {
callbacks: {
label: (ctx) => `${ctx.dataset.label}: ${ctx.parsed.x.toFixed(1)}%`,
...callbacks,
},
},
},
scales: {
x: { min: 0, max: 100, stacked, grid: { color: gridColor() } },
y: {
stacked,
grid: { display: false },
// Every model gets a label, so the type has to fit the row pitch.
ticks: { font: { size: 11 }, autoSkip: false },
},
},
};
}
// Two ways to break a benchmark down into outcomes that are disjoint and cover
// it completely, so a stack is exactly as wide as the benchmark is long.
const FAILURE_VIEWS = {
robloxqa: {
labels: ["Correct", "Wrong", "No answer"],
colors: () => [
cssVar("--color-success"),
cssVar("--color-danger"),
cssVar("--color-text-tertiary"),
],
tally: (m) => {
const total = (key) => add(QA_SPLITS.map((k) => qa(m, k)[key]));
const n = total("n");
const correct = total("correctCount");
const noAnswer = total("noAnswerCount");
return { n, parts: [correct, n - correct - noAnswer, noAnswer] };
},
},
leetcode: {
labels: [
"Fully solved",
"Partial pass",
"Failed all tests",
"Invalid syntax",
"No code",
],
// A severity ramp, then grey for the one outcome that isn't a wrong answer
// so much as an absent one.
colors: () => [
cssVar("--color-success"),
cssVar("--color-warning"),
withAlpha(cssVar("--color-danger"), 0.4),
cssVar("--color-danger"),
cssVar("--color-text-tertiary"),
],
tally: (m) => {
const total = (key) => add(DIFFICULTIES.map((d) => lc(m, d)[key]));
// Partial passes are exclusive of full ones and invalid syntax has been
// taken out of the failures, so these five partition the problem set.
return {
n: total("n"),
parts: [
total("fullPass"),
total("partialPass"),
total("failedTests"),
total("invalidSyntaxCount"),
total("noCode"),
],
};
},
},
};
function paretoFrontier(points) {
// Points where no other point has lower-or-equal cost AND higher-or-equal score.
const sorted = [...points].sort((a, b) => a.x - b.x || b.y - a.y);
const frontier = [];
let best = -Infinity;
for (const p of sorted) {
if (p.y > best) {
frontier.push(p);
best = p.y;
}
}
return frontier;
}
function buildChart(id, config) {
if (charts[id]) charts[id].destroy();
charts[id] = new Chart(document.getElementById(id), config);
if (config.options.plugins?.zoom) attachZoomControls(charts[id]);
}
function renderCharts() {
const models = filteredModels();
// 1. Score vs cost (log x) + Pareto frontier.
const costPoints = models.map((m) => ({
x: m.scores.totalCost,
y: m.scores.overall,
model: m,
}));
buildChart("chart-cost", {
type: "scatter",
data: {
datasets: [
{
type: "line",
data: paretoFrontier(costPoints).map(({ x, y }) => ({ x, y })),
borderColor: withAlpha(cssVar("--color-primary-bright"), 0.5),
borderDash: [6, 5],
borderWidth: 1.5,
pointRadius: 0,
fill: false,
order: 2,
},
{
...scatterDataset(
models,
(m) => m.scores.totalCost,
(m) => m.scores.overall,
),
order: 1,
},
],
},
options: scatterOptions(
"Total cost (USD, log)",
"Composite score",
true,
(x) => `$${x.toFixed(2)}`,
// Zoomed, a tick can land anywhere, so round it: cents below a dollar,
// and no trailing zeros above one.
(x) => (x < 1 ? `$${x.toFixed(2)}` : `$${+x.toFixed(2)}`),
),
});
// 2. Score vs release date. A linear axis over epoch milliseconds does the
// work Chart.js's time scale would, without the date adapter it needs; ticks
// land on quarter starts the same way the size axis lands on powers of two.
const dated = models.filter((m) => m.releaseTs != null);
const releaseOpts = scatterOptions(
"Release date",
"Composite score",
false,
fmtDay, // the tooltip names one model, so it can afford the day
fmtMonth,
);
if (dated.length) {
const times = dated.map((m) => m.releaseTs);
const pad = 1.5 * MONTH_MS; // air at both ends so markers aren't clipped
releaseOpts.scales.x.min = Math.min(...times) - pad;
releaseOpts.scales.x.max = Math.max(...times) + pad;
releaseOpts.scales.x.afterBuildTicks = (axis) => {
axis.ticks = releaseTicks(axis.min, axis.max).map((value) => ({ value }));
};
releaseOpts.scales.x.ticks.autoSkip = false;
if (releaseOpts.plugins.zoom)
releaseOpts.plugins.zoom.limits.x.minRange = RELEASE_MIN_RANGE;
}
// Stepped: a score is the best available from the day it ships until
// something beats it, so the frontier is a ratchet rather than a trend — it
// holds flat and then jumps on release day. That is Chart.js's "before",
// whose naming inverts the usual convention: "after" would climb to the new
// score first and then run right, dating the record before the model shipped.
// It stops at the last model that set a record — running it out to the axis
// edge would draw a line past newer models that did not beat it, which reads
// as a claim that they did.
const frontier = paretoFrontier(
dated.map((m) => ({ x: m.releaseTs, y: m.scores.overall })),
).map(({ x, y }) => ({ x, y }));
buildChart("chart-release", {
type: "scatter",
data: {
datasets: [
{
type: "line",
data: frontier,
stepped: "before",
borderColor: withAlpha(cssVar("--color-primary-bright"), 0.5),
borderDash: [6, 5],
borderWidth: 1.5,
pointRadius: 0,
fill: false,
order: 2,
},
{
...scatterDataset(
dated,
(m) => m.releaseTs,
(m) => m.scores.overall,
),
order: 1,
},
],
},
options: releaseOpts,
});
// 3. Score vs output tokens (log x).
buildChart("chart-tokens", {
type: "scatter",
data: {
datasets: [
scatterDataset(
models,
(m) => m.scores.totalOutputTokens,
(m) => m.scores.overall,
),
],
},
options: scatterOptions(
"Total output tokens (log)",
"Composite score",
true,
(x) => `${x.toLocaleString("en-US")} tokens`,
fmtCompact,
),
});
// 4. Score vs model size (log x) + Pareto frontier; sized models only.
// Ticks sit at power-of-two GiB steps so the 32 GB single-GPU mark always
// lands on a gridline without special treatment.
const sized = models.filter((m) => m.size != null);
const sizePoints = sized.map((m) => ({
x: m.size,
y: m.scores.overall,
}));
const sizeOpts = scatterOptions(
"Model size (log)",
"Composite score",
true,
(x) => fmtBytes(x),
);
if (sized.length) {
const GIB = 1024 ** 3;
const sizes = sized.map((m) => m.size);
const lo = Math.floor(Math.log2(Math.min(...sizes) / GIB));
const hi = Math.ceil(Math.log2(Math.max(...sizes) / GIB));
sizeOpts.scales.x.min = GIB * 2 ** lo;
sizeOpts.scales.x.max = GIB * 2 ** hi;
sizeOpts.scales.x.afterBuildTicks = (axis) => {
// Zoom moves the axis, so the ladder is generated from the range on
// screen rather than the data's. Zoomed in far enough that no power of
// two lands in view, the scale's own ticks beat none at all.
const from = Math.ceil(Math.log2(axis.min / GIB));
const to = Math.floor(Math.log2(axis.max / GIB));
if (to - from < 1) return;
axis.ticks = [];
for (let e = from; e <= to; e++) axis.ticks.push({ value: GIB * 2 ** e });
};
sizeOpts.scales.x.ticks.autoSkip = false;
}
sizeOpts.scales.x.ticks.callback = (v) => fmtBytesTick(v);
buildChart("chart-size", {
type: "scatter",
data: {
datasets: [
{
type: "line",
data: paretoFrontier(sizePoints).map(({ x, y }) => ({ x, y })),
borderColor: withAlpha(cssVar("--color-primary-bright"), 0.5),
borderDash: [6, 5],
borderWidth: 1.5,
pointRadius: 0,
fill: false,
order: 2,
},
{
...scatterDataset(
sized,
(m) => m.size,
(m) => m.scores.overall,
),
order: 1,
},
],
},
options: sizeOpts,
});
// 5. Knowledge vs coding with y = x reference.
const kvcOpts = scatterOptions(
"Knowledge score",
"Coding score",
false,
(x) => `${x.toFixed(1)} knowledge`,
);
kvcOpts.scales.x.min = 0;
kvcOpts.scales.x.max = 100;
kvcOpts.scales.y.min = 0;
kvcOpts.scales.y.max = 100;
buildChart("chart-kvc", {
type: "scatter",
data: {
datasets: [
{
type: "line",
data: [
{ x: 0, y: 0 },
{ x: 100, y: 100 },
],
borderColor: withAlpha(cssVar("--color-text-tertiary"), 0.4),
borderDash: [6, 5],
borderWidth: 1,
pointRadius: 0,
order: 2,
},
{
...scatterDataset(
models,
(m) => m.scores.knowledge,
(m) => m.scores.coding,
),
order: 1,
},
],
},
options: kvcOpts,
});
// 6. Where the score comes from: the five weighted terms of the composite,
// stacked. Every bar ends at the model's Score column, which is the point —
// the methodology card states the weights, this shows them doing the work.
const byOverall = [...models].sort(
(a, b) => b.scores.overall - a.scores.overall,
);
const accent = cssVar("--color-accent");
const purple = cssVar("--color-primary-bright");
// Two shades of accent for knowledge, three of purple for coding, so a bar
// reads as two families before it reads as five terms. Weight order runs
// faint to solid within each.
const termColors = [
accent,
withAlpha(accent, 0.5),
withAlpha(purple, 0.4),
withAlpha(purple, 0.68),
purple,
];
buildChart("chart-decomp", {
type: "bar",
data: {
labels: byOverall.map(shortName),
datasets: state.data.models[0].scores.contributions.map(
({ label }, i) => ({
label,
data: byOverall.map((m) => m.scores.contributions[i].points),
backgroundColor: termColors[i],
borderRadius: 2,
barPercentage: 0.9,
categoryPercentage: 0.8,
}),
),
},
options: rowChartOptions({
stacked: true,
callbacks: {
label: (ctx) => {
const term =
byOverall[ctx.dataIndex].scores.contributions[ctx.datasetIndex];
return `${ctx.dataset.label}: ${term.points.toFixed(1)} pts (from ${term.from.toFixed(1)}%)`;
},
footer: (items) =>
`Composite: ${byOverall[items[0].dataIndex].scores.overall.toFixed(1)}`,
},
}),
});
renderTopicsChart(models);
renderFailureChart(models);
renderDifficultyChart(models);
}
// The three charts below answer to a chip strip of their own. Each is its own
// function so pressing a chip rebuilds that one chart instead of tearing down
// and recreating all nine — which is both wasted work and a chance for the
// page to shift under you while eight unrelated canvases are resized.
// 7. The knowledge half of the decomposition bar, opened up by docs area. Same
// stacking rule, so the two charts read as a drill-down: every bar here ends at
// the Knowledge column the way every bar above ends at the Score column.
function renderTopicsChart(models) {
const topicTerms =
state.data.models[0].scores.topicContributions[state.topicSplit];
document.getElementById("chart-topics").closest(".chart-card").hidden =
topicTerms.length === 0;
if (topicTerms.length) {
const topicTotal = (m) =>
add(m.scores.topicContributions[state.topicSplit].map((t) => t.points));
const byKnowledge = [...models].sort(
(a, b) => topicTotal(b) - topicTotal(a),
);
// Ten ordered categories are past what the site's two-colour palette can
// keep apart — shades of one hue blur together and alternating two hues
// reads as stripes. Even hue steps at a fixed saturation and lightness
// separate all ten, and give no segment more visual weight than its
// neighbours. Lightness follows the theme so each stays legible against
// the card behind it.
// Taken in order the hues arrive as a gradient, and neighbours a single step
// apart are the hardest pairs to tell apart — exactly the pairs that share
// an edge. So walk the wheel in strides instead, which visits every hue but
// puts a third of the wheel between each segment and the next. The stride
// has to be coprime with the count or the walk cycles through a few hues
// and repeats them.
const coprime = (a, b) => {
while (b) [a, b] = [b, a % b];
return a === 1;
};
let stride = Math.max(1, Math.round(topicTerms.length / 3));
while (!coprime(stride, topicTerms.length)) stride++;
const lightTheme = document.documentElement.dataset.theme === "light";
const topicColors = topicTerms.map((_, i) => {
const slot = (i * stride) % topicTerms.length;
const hue = Math.round(265 + (360 * slot) / topicTerms.length) % 360;
return `hsl(${hue}, ${lightTheme ? 55 : 62}%, ${lightTheme ? 44 : 62}%)`;
});
buildChart("chart-topics", {
type: "bar",
data: {
labels: byKnowledge.map(shortName),
datasets: topicTerms.map(({ label }, i) => ({
label,
data: byKnowledge.map(
(m) => m.scores.topicContributions[state.topicSplit][i].points,
),
backgroundColor: topicColors[i],
borderRadius: 2,
barPercentage: 0.9,
categoryPercentage: 0.8,
})),
},
options: rowChartOptions({
stacked: true,
callbacks: {
label: (ctx) => {
const t =
byKnowledge[ctx.dataIndex].scores.topicContributions[
state.topicSplit
][ctx.datasetIndex];
return `${ctx.dataset.label}: ${t.points.toFixed(1)} pts (${t.from.toFixed(1)}% of ${fmtTokens(t.n)} questions)`;
},
footer: (items) =>
`${TOPIC_VIEWS[state.topicSplit].label}: ${topicTotal(byKnowledge[items[0].dataIndex]).toFixed(1)}`,
},
}),
});
}
}
// 8. How a model fails, not just how often — the difference between answering
// wrong and never answering is invisible in a single accuracy column.
function renderFailureChart(models) {
const view = FAILURE_VIEWS[state.failureBench];
const tallies = models
.map((m) => ({ model: m, ...view.tally(m) }))
.sort((a, b) => b.parts[0] / b.n - a.parts[0] / a.n);
const outcomeColors = view.colors();
buildChart("chart-failure", {
type: "bar",
data: {
labels: tallies.map((t) => shortName(t.model)),
datasets: view.labels.map((label, i) => ({
label,
data: tallies.map((t) => pct(t.parts[i], t.n)),
backgroundColor: outcomeColors[i],
borderRadius: 2,
barPercentage: 0.9,
categoryPercentage: 0.8,
})),
},
options: rowChartOptions({
stacked: true,
callbacks: {
label: (ctx) => {
const t = tallies[ctx.dataIndex];
return `${ctx.dataset.label}: ${fmtTokens(t.parts[ctx.datasetIndex])} of ${fmtTokens(t.n)} (${ctx.parsed.x.toFixed(1)}%)`;
},
},
}),
});
}
// 9. Difficulty breakdown: horizontal grouped bars.
function renderDifficultyChart(models) {
const metric = state.difficultyMetric;
const byScore = [...models].sort(
(a, b) => lc(b, "overall")[metric] - lc(a, "overall")[metric],
);
const diffColors = {
easy: cssVar("--color-success"),
medium: cssVar("--color-warning"),
hard: cssVar("--color-danger"),
};
buildChart("chart-difficulty", {
type: "bar",
data: {
labels: byScore.map(shortName),
datasets: DIFFICULTIES.map((d) => ({
label: d,
data: byScore.map((m) => lc(m, d)[metric]),
backgroundColor: diffColors[d],
borderRadius: 2,
barPercentage: 0.85,
categoryPercentage: 0.72,
})),
},
options: rowChartOptions(),
});
}
/* ---------- Events ---------- */
function rerender() {
renderTable();
renderCharts();
}
function setChipGroup(groupEl, value) {
groupEl
.querySelectorAll(".chip")
.forEach((c) => c.classList.toggle("is-active", c.dataset.value === value));
}
function wireEvents() {
document.getElementById("theme-toggle").addEventListener("click", () => {
const next =
document.documentElement.dataset.theme === "dark" ? "light" : "dark";
document.documentElement.dataset.theme = next;
try {
localStorage.setItem("theme", next);
} catch (e) {}
// Chart colors are resolved from CSS tokens at build time, so rebuild everything.
chartDefaults();
rerender();
// Markers bake in the theme's ring color, so they are redrawn out of band.
buildProviderMarkers().then(renderCharts);
});
// devicePixelRatio moves when the page is zoomed or the window is dragged to
// another monitor, and both the canvases and the marker bitmaps are baked at
// the old one. A media query is pinned to the ratio it was built with, so
// each firing has to re-arm against the new value.
const watchPixelRatio = () => {
matchMedia(`(resolution: ${window.devicePixelRatio}dppx)`).addEventListener(
"change",
() => {
chartDefaults();
buildProviderMarkers().then(renderCharts);
watchPixelRatio();
},
{ once: true },
);
};
watchPixelRatio();
document.getElementById("search").addEventListener("input", (e) => {
state.search = e.target.value;
rerender();
});
// Filters live above both views, so they redraw the charts as well as the table.
document.getElementById("weights-filter").addEventListener("click", (e) => {
const chip = e.target.closest(".chip");
if (!chip) return;
state.weightsFilter = chip.dataset.value;
setChipGroup(e.currentTarget, state.weightsFilter);
rerender();
});
document
.querySelector(".table-scroll")
.addEventListener("scroll", updateScrollShadows, { passive: true });
window.addEventListener("resize", updateScrollShadows);
// Column-set tabs
document.querySelectorAll(".board-tab").forEach((tab) => {
tab.addEventListener("click", () => {
state.columnSet = tab.dataset.set;
document
.querySelectorAll(".board-tab")
.forEach((t) => t.classList.toggle("is-active", t === tab));
document.getElementById("difficulty-select").hidden =
state.columnSet !== "leetcode";
document.getElementById("qa-split-select").hidden =
state.columnSet !== "robloxqa";
if (!activeColumns().some((c) => c.key === state.sortKey)) {
state.sortKey = activeColumns()[0].key;
state.sortDir = -1;
}
renderTable();
});
});
// Each benchmark tab carries a chip strip picking which slice of it the table
// shows. A QA split swaps columns outright, so re-home the sort if its column
// just left.
const wireViewSelect = (id, stateKey) =>
document.getElementById(id).addEventListener("click", (e) => {
const chip = e.target.closest(".chip");
if (!chip) return;
state[stateKey] = chip.dataset.value;
setChipGroup(e.currentTarget, state[stateKey]);
if (!activeColumns().some((c) => c.key === state.sortKey)) {
state.sortKey = activeColumns()[0].key;
state.sortDir = -1; // a new column, so the old direction means nothing
}
renderTable();
});
wireViewSelect("difficulty-select", "difficulty");
wireViewSelect("qa-split-select", "qaSplit");
// Sorting + row expansion (delegated)
document
.querySelector("#board-table thead")
.addEventListener("click", (e) => {
const th = e.target.closest("th[data-key]");
if (!th) return;
const key = th.dataset.key;
if (state.sortKey === key) state.sortDir *= -1;
else {
state.sortKey = key;
state.sortDir = -1;
}
renderTable();
});
document
.querySelector("#board-table tbody")
.addEventListener("click", (e) => {
if (e.target.closest("a")) return; // let model links work
const row = e.target.closest("tr.model-row");
if (!row) return;
state.expandedId =
state.expandedId === row.dataset.id ? null : row.dataset.id;
renderTable();
});
document
.getElementById("difficulty-metric")
.addEventListener("click", (e) => {
const chip = e.target.closest(".chip");
if (!chip) return;
state.difficultyMetric = chip.dataset.value;
setChipGroup(e.currentTarget, state.difficultyMetric);
document.getElementById("difficulty-caption").textContent =
state.difficultyMetric === "fullySolved"
? "Problems fully solved on Easy / Medium / Hard."
: "Average unit tests passed on Easy / Medium / Hard.";
renderDifficultyChart(filteredModels());
});
document.getElementById("topic-split").addEventListener("click", (e) => {
const chip = e.target.closest(".chip");
if (!chip) return;
state.topicSplit = chip.dataset.value;
setChipGroup(e.currentTarget, state.topicSplit);
// Each view totals a different published number, so say which one.
document.getElementById("topic-caption").textContent =
state.topicSplit === "overall"
? "Each bar ends at the model's Knowledge score."
: `Each bar ends at the model's ${TOPIC_VIEWS[state.topicSplit].label.toLowerCase()} accuracy.`;
renderTopicsChart(filteredModels());
});
document.getElementById("failure-bench").addEventListener("click", (e) => {
const chip = e.target.closest(".chip");
if (!chip) return;
state.failureBench = chip.dataset.value;
setChipGroup(e.currentTarget, state.failureBench);
document.getElementById("failure-caption").textContent =
state.failureBench === "robloxqa"
? "Breakdown of outcomes across both splits."
: "Breakdown of outcomes across all three difficulties.";
renderFailureChart(filteredModels());
});
}
/* ---------- Sonar sweep beam (ported from the brand site's SonarBackground.tsx) ---------- */
// Beam color stops (offset, rgba-hex), mirrored from the original CSS conic
// gradient: transparent until ~86%, ramping to a bright leading edge just
// before the seam, then transparent at the wrap so there's no visible join.
const BEAM_STOPS = [
[0, "#00000000"],
[0.86, "#00000000"],
[0.92, "#3224A514"],
[0.97, "#3224A547"],
[0.995, "#3224A5CC"],
[1, "#00000000"],
];
// Bake the beam's conic gradient into a data-URL bitmap ONCE. A live CSS
// conic-gradient is re-rasterized every frame as the element rotates on mobile
// WebKit (an atan2 per pixel), which stalled the sweep to single-digit FPS. A
// raster texture, by contrast, is something the GPU caches and simply spins.
// The beam is soft, so a modest fixed resolution scales up cleanly to any
// field size. Returns null when canvas / conic gradients are unavailable, in
// which case the sweep renders nothing.
function renderBeamTexture() {
try {
const size = 600;
const canvas = document.createElement("canvas");
canvas.width = size;
canvas.height = size;
const ctx = canvas.getContext("2d");
if (!ctx || typeof ctx.createConicGradient !== "function") return null;
// Start at the top to match the CSS `from 0deg`. Continuous rotation makes
// the absolute start angle moot, but it keeps the baked image faithful.
const gradient = ctx.createConicGradient(-Math.PI / 2, size / 2, size / 2);
for (const [offset, color] of BEAM_STOPS)
gradient.addColorStop(offset, color);
ctx.fillStyle = gradient;
ctx.fillRect(0, 0, size, size);
// Carve the soft circular vignette into the texture's alpha (the same
// radial falloff the grid gets from its sonar-fade stroke).
// `destination-in` multiplies the existing pixels by this radial alpha, so
// the beam dissolves smoothly near the rim instead of ending at the hard
// edge of the circular clip, and being baked in it costs nothing per frame.
ctx.globalCompositeOperation = "destination-in";
const fade = ctx.createRadialGradient(
size / 2,
size / 2,
0,
size / 2,
size / 2,
size / 2,
);
fade.addColorStop(0, "#000000ff");
fade.addColorStop(0.2, "#000000ff");
fade.addColorStop(0.95, "#00000000");
ctx.fillStyle = fade;
ctx.fillRect(0, 0, size, size);
return canvas.toDataURL();
} catch {
return null;
}
}
function initSonar() {
const field = document.querySelector(".sonar-field");
if (!field) return;
const texture = renderBeamTexture();
if (!texture) return;
const beam = document.createElement("div");
beam.className = "sonar-beam";
beam.style.backgroundImage = `url(${texture})`;
// Beam sits above the grid, below the pings, matching the original order.
field.insertBefore(beam, field.querySelector(".sonar-ping-pos"));
}
/* ---------- Init ---------- */
// Every row of the table is built from a fetch, so at first paint the page is
// well short of the height it ends up with. Reload while scrolled past the
// board and the browser restores the old position against that short page, then
// corrects itself once the rows land — and it can hold that correction back
// until the next layout change, which is whatever you click first. The result
// is a click on a chart chip that shoves the view down by the height of the
// table. Take the restore over instead: remember where we were, and put it back
// once the content that gives the page its height is actually in the DOM.
const SCROLL_KEY = "scroll-y";
function keepScrollPosition() {
if (!("scrollRestoration" in history)) return;
history.scrollRestoration = "manual";
// pagehide rather than unload: it fires for the back/forward cache too.
addEventListener("pagehide", () => {
try {
sessionStorage.setItem(SCROLL_KEY, String(Math.round(window.scrollY)));
} catch (e) {}
});
}
function restoreScrollPosition() {
let saved = null;
try {
saved = sessionStorage.getItem(SCROLL_KEY);
sessionStorage.removeItem(SCROLL_KEY);
} catch (e) {}
// A link straight to a section outranks wherever we were last time.
if (saved == null || location.hash) return;
// Only stand in for the browser where it would have restored the position
// itself. Arriving by link or by typing the address is a fresh visit and
// belongs at the top, whatever this tab was looking at earlier.
const nav = performance.getEntriesByType("navigation")[0];
if (nav && nav.type !== "reload" && nav.type !== "back_forward") return;
// "instant" so the site's smooth scrolling doesn't animate the jump back.
window.scrollTo({ top: Number(saved), behavior: "instant" });
}
async function init() {
const res = await fetch("data/leaderboard.json");
state.data = await res.json();
deriveScores(state.data);
document.getElementById("generated-at").textContent = state.data.generatedAt;
renderEvalCostTotal();
chartDefaults();
wireEvents();
rerender();
// The rows exist now, so the page is finally as tall as it is going to get.
restoreScrollPosition();
// The board doesn't wait on artwork: points start as plain discs and pick up
// their provider's logo as soon as the files decode.
await loadProviderLogos();
await buildProviderMarkers();
renderCharts();
}
keepScrollPosition(); // before the browser gets a chance to restore it itself
initSonar();
init();