Roblox-LLM-Leaderboard

Running

App Files Files Community

Roblox-LLM-Leaderboard / index.html

boatbomber

Mention the number of knowledge questions

a6655b5 7 days ago

Raw

History Blame Contribute Delete

17.8 kB

	<!doctype html>
	<html lang="en" data-theme="dark">
	<head>
	<meta charset="utf-8" />
	<meta name="viewport" content="width=device-width, initial-scale=1" />
	<title>Roblox LLM Leaderboard</title>
	<meta
	name="description"
	content="Evaluating LLMs on Roblox gamedev knowledge and Luau code generation."
	/>
	<meta name="theme-color" content="#3325A4" />
	<link rel="icon" type="image/svg+xml" href="assets/brand/logo.svg" />
	<script>
	// Apply saved theme before first paint.
	try {
	const t = localStorage.getItem("theme");
	if (t === "light" \|\| t === "dark")
	document.documentElement.dataset.theme = t;
	} catch (e) {}
	</script>
	<link rel="preconnect" href="https://fonts.googleapis.com" />
	<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
	<link
	href="https://fonts.googleapis.com/css2?family=Big+Shoulders+Display:wght@400;500;600&family=Inter:wght@400;500;600&display=swap"
	rel="stylesheet"
	/>
	<link rel="stylesheet" href="assets/style.css" />
	<script src="https://cdn.jsdelivr.net/npm/chart.js@4/dist/chart.umd.min.js"></script>
	</head>
	<body>
	<nav class="site-nav">
	<div class="wrap nav-inner">
	<a class="nav-brand" href="https://www.torpedosoftware.llc/">
	<img
	class="wordmark wordmark-light"
	src="assets/brand/wordmark-light.svg"
	alt="Torpedo Software"
	height="100%"
	/>
	<img
	class="wordmark wordmark-dark"
	src="assets/brand/wordmark-dark.svg"
	alt="Torpedo Software"
	height="100%"
	/>
	</a>
	<div class="nav-links">
	<a href="https://huggingface.co/TorpedoSoftware">Hugging Face</a>
	<button
	id="theme-toggle"
	class="theme-toggle"
	aria-label="Toggle color theme"
	>
	<span class="theme-icon-dark">◐</span
	><span class="theme-icon-light">◑</span>
	</button>
	</div>
	</div>
	</nav>

	<header class="site-header">
	<!-- Naval-sonar field ported verbatim from the brand site's
	SonarBackground.tsx: static grid as cached SVG primitives (CSS
	repeating-gradients re-rasterize every repaint), soft vignette as a
	radial-gradient stroke (a CSS mask forces an offscreen pass per
	frame), and the sweep beam baked to a bitmap by app.js (a live
	conic-gradient is re-rasterized every frame as it rotates). -->
	<div class="sonar-root" aria-hidden="true">
	<div class="sonar-field">
	<svg
	class="sonar-grid"
	viewBox="0 0 100 100"
	preserveAspectRatio="xMidYMid meet"
	fill="none"
	stroke="url(#sonar-fade)"
	>
	<defs>
	<!-- Alpha falls off with radius to fade the field into the
	canvas; currentColor stops keep it theme-reactive. -->
	<radialGradient
	id="sonar-fade"
	gradientUnits="userSpaceOnUse"
	cx="50"
	cy="50"
	r="70.71"
	>
	<stop offset="0" stop-color="currentColor" stop-opacity="1" />
	<stop offset="0.2" stop-color="currentColor" stop-opacity="1" />
	<stop offset="0.8" stop-color="currentColor" stop-opacity="0" />
	</radialGradient>
	</defs>
	<!-- Range rings: 5-unit steps, past the 50-unit edge so rings
	still reach the square's corners. -->
	<g class="sonar-ring">
	<circle
	cx="50"
	cy="50"
	r="5"
	vector-effect="non-scaling-stroke"
	/>
	<circle
	cx="50"
	cy="50"
	r="10"
	vector-effect="non-scaling-stroke"
	/>
	<circle
	cx="50"
	cy="50"
	r="15"
	vector-effect="non-scaling-stroke"
	/>
	<circle
	cx="50"
	cy="50"
	r="20"
	vector-effect="non-scaling-stroke"
	/>
	<circle
	cx="50"
	cy="50"
	r="25"
	vector-effect="non-scaling-stroke"
	/>
	<circle
	cx="50"
	cy="50"
	r="30"
	vector-effect="non-scaling-stroke"
	/>
	<circle
	cx="50"
	cy="50"
	r="35"
	vector-effect="non-scaling-stroke"
	/>
	<circle
	cx="50"
	cy="50"
	r="40"
	vector-effect="non-scaling-stroke"
	/>
	<circle
	cx="50"
	cy="50"
	r="45"
	vector-effect="non-scaling-stroke"
	/>
	<circle
	cx="50"
	cy="50"
	r="50"
	vector-effect="non-scaling-stroke"
	/>
	<circle
	cx="50"
	cy="50"
	r="55"
	vector-effect="non-scaling-stroke"
	/>
	<circle
	cx="50"
	cy="50"
	r="60"
	vector-effect="non-scaling-stroke"
	/>
	<circle
	cx="50"
	cy="50"
	r="65"
	vector-effect="non-scaling-stroke"
	/>
	<circle
	cx="50"
	cy="50"
	r="70"
	vector-effect="non-scaling-stroke"
	/>
	</g>
	<!-- 12 spokes every 30deg, as 6 full diameters through center. -->
	<g class="sonar-spoke">
	<line
	x1="50"
	y1="125"
	x2="50"
	y2="-25"
	vector-effect="non-scaling-stroke"
	/>
	<line
	x1="12.5"
	y1="114.952"
	x2="87.5"
	y2="-14.952"
	vector-effect="non-scaling-stroke"
	/>
	<line
	x1="-14.952"
	y1="87.5"
	x2="114.952"
	y2="12.5"
	vector-effect="non-scaling-stroke"
	/>
	<line
	x1="-25"
	y1="50"
	x2="125"
	y2="50"
	vector-effect="non-scaling-stroke"
	/>
	<line
	x1="-14.952"
	y1="12.5"
	x2="114.952"
	y2="87.5"
	vector-effect="non-scaling-stroke"
	/>
	<line
	x1="12.5"
	y1="-14.952"
	x2="87.5"
	y2="114.952"
	vector-effect="non-scaling-stroke"
	/>
	</g>
	<!-- Crosshair: brighter vertical + horizontal over the spokes. -->
	<g class="sonar-crosshair">
	<line
	x1="50"
	y1="-25"
	x2="50"
	y2="125"
	vector-effect="non-scaling-stroke"
	/>
	<line
	x1="-25"
	y1="50"
	x2="125"
	y2="50"
	vector-effect="non-scaling-stroke"
	/>
	</g>
	</svg>
	<!-- Sweep beam div injected here by app.js once its texture bakes. -->
	<!-- Pings: wrapper owns position/centering, inner span owns the
	animated transform, so the two never fight over `transform`.
	Delays land each bloom as the 8s beam passes; 16s/24s periods
	fire every other/third sweep so contacts read intermittent. -->
	<span
	class="sonar-ping-pos"
	style="
	left: 67.8%;
	top: 24.6%;
	width: 8px;
	height: 8px;
	margin-left: -4px;
	margin-top: -4px;
	"
	>
	<span
	class="sonar-ping"
	style="animation-delay: 0.78s; animation-duration: 16s"
	></span>
	</span>
	<span
	class="sonar-ping-pos"
	style="
	left: 69.7%;
	top: 46.5%;
	width: 6px;
	height: 6px;
	margin-left: -3px;
	margin-top: -3px;
	"
	>
	<span
	class="sonar-ping"
	style="animation-delay: 1.78s; animation-duration: 8s"
	></span>
	</span>
	<span
	class="sonar-ping-pos"
	style="
	left: 76.8%;
	top: 72.5%;
	width: 9px;
	height: 9px;
	margin-left: -4.5px;
	margin-top: -4.5px;
	"
	>
	<span
	class="sonar-ping"
	style="animation-delay: 2.89s; animation-duration: 24s"
	></span>
	</span>
	<span
	class="sonar-ping-pos"
	style="
	left: 43.7%;
	top: 63.6%;
	width: 6px;
	height: 6px;
	margin-left: -3px;
	margin-top: -3px;
	"
	>
	<span
	class="sonar-ping"
	style="animation-delay: 4.56s; animation-duration: 8s"
	></span>
	</span>
	<span
	class="sonar-ping-pos"
	style="
	left: 23.4%;
	top: 57.1%;
	width: 8px;
	height: 8px;
	margin-left: -4px;
	margin-top: -4px;
	"
	>
	<span
	class="sonar-ping"
	style="animation-delay: 5.67s; animation-duration: 16s"
	></span>
	</span>
	<span
	class="sonar-ping-pos"
	style="
	left: 31.6%;
	top: 34.6%;
	width: 7px;
	height: 7px;
	margin-left: -3.5px;
	margin-top: -3.5px;
	"
	>
	<span
	class="sonar-ping"
	style="animation-delay: 6.89s; animation-duration: 24s"
	></span>
	</span>
	</div>
	</div>
	<div class="wrap">
	<p class="eyebrow">Torpedo Software // Evals</p>
	<h1>Roblox LLM <span class="accent">Leaderboard</span></h1>
	<p class="lede">
	Sounding out which models actually know Roblox, and what they cost to
	run. Every model here is benchmarked on
	<a
	href="https://huggingface.co/datasets/TorpedoSoftware/RobloxQA-v2.0"
	>RobloxQA v2.0</a
	>
	(Roblox & Luau knowledge) and
	<a href="https://huggingface.co/datasets/TorpedoSoftware/LuauLeetcode"
	>Luau Leetcode</a
	>
	(single-shot code generation, 208 problems).
	</p>
	</div>
	</header>

	<main class="wrap">
	<section class="controls" aria-label="Search">
	<input
	id="search"
	class="search"
	type="search"
	placeholder="Filter models or providers…"
	autocomplete="off"
	/>
	</section>

	<section class="board" aria-label="Leaderboard">
	<div class="board-tabs" role="tablist">
	<button class="board-tab is-active" role="tab" data-set="overview">
	Overview
	</button>
	<button class="board-tab" role="tab" data-set="robloxqa">
	RobloxQA
	</button>
	<button class="board-tab" role="tab" data-set="leetcode">
	Luau Leetcode
	</button>
	<div class="difficulty-select" id="difficulty-select" hidden>
	<button class="chip is-active" data-value="overall">Overall</button>
	<button class="chip" data-value="easy">Easy</button>
	<button class="chip" data-value="medium">Medium</button>
	<button class="chip" data-value="hard">Hard</button>
	</div>
	</div>
	<div class="table-scroll">
	<table id="board-table">
	<thead></thead>
	<tbody></tbody>
	</table>
	</div>
	<p class="board-note">
	Click a column to sort, click a row for details.
	</p>
	</section>

	<section class="charts" aria-label="Charts">
	<h2>The trade-off space</h2>
	<p class="section-lede">
	Points are colored by provider. Hover for details, click to open the
	model's page.
	</p>
	<div class="chart-grid">
	<figure class="chart-card">
	<figcaption>
	<strong>Score vs cost</strong>
	<span>Composite score against total eval spend (log scale).</span>
	</figcaption>
	<div class="chart-box"><canvas id="chart-cost"></canvas></div>
	</figure>
	<figure class="chart-card">
	<figcaption>
	<strong>Score vs output tokens</strong>
	<span
	>Composite score against total output tokens (log scale).</span
	>
	</figcaption>
	<div class="chart-box"><canvas id="chart-tokens"></canvas></div>
	</figure>
	<figure class="chart-card">
	<figcaption>
	<strong>Score vs model size</strong>
	<span
	>Composite score against weight size on disk (log scale). Models
	without published weights are omitted.</span
	>
	</figcaption>
	<div class="chart-box"><canvas id="chart-size"></canvas></div>
	</figure>
	<figure class="chart-card">
	<figcaption>
	<strong>Knowledge vs coding</strong>
	<span
	>Weighted knowledge score against weighted coding score.</span
	>
	</figcaption>
	<div class="chart-box"><canvas id="chart-kvc"></canvas></div>
	</figure>
	<figure class="chart-card">
	<figcaption>
	<strong>Leetcode by difficulty</strong>
	<span id="difficulty-caption"
	>Problems fully solved on Easy / Medium / Hard.</span
	>
	<span class="metric-toggle chip-group" id="difficulty-metric">
	<button class="chip is-active" data-value="fullySolved">
	Fully solved
	</button>
	<button class="chip" data-value="unitTestsPassed">
	Unit tests
	</button>
	</span>
	</figcaption>
	<div class="chart-box tall">
	<canvas id="chart-difficulty"></canvas>
	</div>
	</figure>
	</div>
	</section>

	<section class="about" aria-label="Methodology">
	<h2>Methodology</h2>
	<div class="about-grid">
	<div>
	<h3>RobloxQA v2.0</h3>
	<p>
	Knowledge testing: 3,000 questions about Roblox and Luau, scored
	both as freeform answers (judged) and as multiple choice.
	<a
	href="https://huggingface.co/datasets/TorpedoSoftware/RobloxQA-v2.0"
	>Dataset ↗</a
	>
	</p>
	</div>
	<div>
	<h3>Luau Leetcode</h3>
	<p>
	Single-shot code generation on 208 Leetcode problems translated to
	Luau (47 easy, 97 medium, 64 hard), run against unit tests with
	syntax validation and code-quality scoring.
	<a
	href="https://huggingface.co/datasets/TorpedoSoftware/LuauLeetcode"
	>Dataset ↗</a
	>
	</p>
	</div>
	<div>
	<h3>Composite score</h3>
	<p>
	Half knowledge, half coding, with weights favoring the harder
	signals. Knowledge weighs freeform answers over multiple choice:
	<code>0.7 × freeform + 0.3 × multiple choice</code>. Coding
	scores each difficulty as
	<code
	>0.4 × unit tests + 0.25 × fully solved + 0.15 ×
	partially solved + 0.1 × code quality + 0.1 ×
	valid syntax</code
	>, then blends difficulties as
	<code>0.2 × easy + 0.35 × medium + 0.45 × hard</code>.
	</p>
	</div>
	<div>
	<h3>Cost</h3>
	<p>
	Based on OpenRouter's effective pricing. Torpedo Software models
	not on OpenRouter were run on an RX 7900XTX, with cost computed
	from electricity use at 20 cents per kilowatt-hour.
	</p>
	</div>
	</div>
	</section>
	</main>

	<footer class="site-footer">
	<div class="wrap">
	<p>
	Built by <a href="https://huggingface.co/boatbomber">boatbomber</a> ·
	<a href="https://www.torpedosoftware.llc/">Torpedo Software</a> · MIT
	license · Evals generated <span id="generated-at"></span>
	</p>
	</div>
	</footer>

	<script src="assets/app.js"></script>
	</body>
	</html>