Hugging Face's logo

Spaces:
SaylorTwift
/
benchmark-holo-deck
Running

App Files Community
benchmark-holo-deck / index.html
SaylorTwift's picture
SaylorTwift HF Staff
Upload folder using huggingface_hub
a1f868f verified
Raw
History Blame Contribute Delete
166 kB
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Benchmark Holo Deck</title>
<style>
:root {
 --bg: #0d0e12;
 --bg2: #16171d;
 --ink: #f2f1ec;
 --muted: #9a9890;
 --hair: #2a2b33;
 --accent: #eda100;
 /* self-generated fractal-noise grain (we don't have the real project's
 grain PNG asset) β€” same role as their --grain: a subtle foil texture
 for the Secret Rare tier, not a decorative background */
 --grain: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='180' height='180'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='0.85' numOctaves='2' stitchTiles='stitch'/%3E%3CfeColorMatrix type='matrix' values='0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0.5 0'/%3E%3C/filter%3E%3Crect width='100%25' height='100%25' filter='url(%23n)'/%3E%3C/svg%3E");
}
* { box-sizing: border-box; }
html, body {
 margin: 0; padding: 0; background: var(--bg); color: var(--ink);
 font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
 min-height: 100vh;
}
/* ---------------------------------------------------------------- hero */
.hero { max-width: 1200px; margin: 0 auto; padding: 36px 24px 8px; }
.hero h1 {
 font-size: 30px; margin: 0 0 6px;
 background: linear-gradient(90deg in oklch, #ff7a7a, #ffd27a, #7affb2, #7ac8ff, #c58bff, #ff7ad4);
 -webkit-background-clip: text; background-clip: text; color: transparent;
 background-size: 300% 100%; animation: hue-drift 10s linear infinite;
}
@keyframes hue-drift { to { background-position: 300% 0; } }
.hero p { color: var(--muted); font-size: 14px; margin: 0 0 4px; max-width: 720px; line-height: 1.6; }
.hero a { color: #7ac8ff; }
.hero .note { font-size: 12.5px; color: var(--muted); margin-top: 8px; }
/* ---------------------------------------------------------------- controls */
.controls {
 max-width: 1200px; margin: 18px auto 6px; padding: 0 24px;
 display: flex; flex-wrap: wrap; gap: 14px; align-items: center;
}
.pillrow { display: flex; flex-wrap: wrap; gap: 6px; }
.pill {
 border: 1px solid var(--hair); background: var(--bg2); color: var(--ink);
 border-radius: 999px; padding: 6px 13px; font-size: 12.5px; cursor: pointer;
 transition: border-color .15s, background .15s;
}
.pill:hover { border-color: var(--pill-c, var(--accent)); }
.pill.on { background: var(--pill-c, var(--accent)); border-color: var(--pill-c, var(--accent)); color: #0d0e12; font-weight: 600; }
.spacer { flex: 1; }
select.sortsel, input.search {
 background: var(--bg2); color: var(--ink); border: 1px solid var(--hair);
 border-radius: 8px; padding: 7px 12px; font-size: 13px;
}
select.sortsel:focus, input.search:focus { outline: none; border-color: var(--accent); }
.count-note { max-width: 1200px; margin: 0 auto; padding: 4px 24px 0; color: var(--muted); font-size: 12.5px; }
/* ---------------------------------------------------------------- grid */
.grid {
 max-width: 1200px; margin: 0 auto; padding: 18px 24px 60px;
 display: grid; grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); gap: 26px;
}
/* ---------------------------------------------------------------- card */
.card-wrap { perspective: 900px; }
.card {
 /* rarity ladder, indexed by how many major labs' *latest* release use this
 benchmark (0-6). Tier 0 = Common: flat card, no effects at all. Every
 property below ramps up tier by tier; .card--rN classes only override
 these β€” the actual effect rules (foil/glare/border) read the variables
 generically so there's no per-tier rule duplication. */
 --foil-op: 0; --glare-op: 0;
 --border-w: 1px; --rest-glow: 0px; --pulse: none;
 position: relative; aspect-ratio: 5 / 7; border-radius: 16px; z-index: 1;
 background: var(--card-bg); cursor: pointer;
 transform: rotateX(0deg) rotateY(0deg) scale(1);
 transition: transform .5s cubic-bezier(.23,1,.32,1), box-shadow .5s;
 transform-style: preserve-3d;
 box-shadow: 0 6px 18px rgba(0,0,0,.5), 0 0 var(--rest-glow) var(--glow, transparent), 0 0 0 1px rgba(255,255,255,.05) inset;
 isolation: isolate;
 animation: var(--pulse);
}
.card.live { transition: transform .08s linear, box-shadow .3s; }
.card:hover, .card.live {
 box-shadow: 0 22px 40px -12px rgba(0,0,0,.65), 0 0 calc(var(--rest-glow) + 14px) var(--glow, transparent), 0 0 0 1px rgba(255,255,255,.08) inset;
}
/* Common (0 labs): nothing further β€” plain card, tilt only.
 Uncommon (1): "reverse holo" style β€” a grayscale satin sheen, no rainbow.
 Rare (2) / Rare Holo (3): the real "regular holo" rainbow+scanline+bars
 treatment, at increasing opacity/glow.
 Ultra Rare (4): brighter still, pulsing glow.
 Rainbow Rare (5): + animated rainbow border/badge, faster pulse.
 Secret Rare (6, "V"): the real PokΓ©mon-V mirrored-band treatment. */
.card--r1 { --border-w: 1px; }
.card--r2 { --foil-op: .8; --glare-op: .8; --border-w: 1.5px; }
.card--r3 { --foil-op: 1; --glare-op: .9; --border-w: 2px; --rest-glow: 16px; }
.card--r4 { --foil-op: 1; --glare-op: 1; --border-w: 2.5px; --rest-glow: 22px; --pulse: pulse-glow 3.2s ease-in-out infinite; }
.card--r5 { --foil-op: 1; --glare-op: 1; --border-w: 3px; --rest-glow: 28px; --pulse: pulse-glow 2.4s ease-in-out infinite; }
.card--r6 { --foil-op: .55; --glare-op: .55; --border-w: 3.5px; --rest-glow: 36px; --pulse: pulse-glow 1.7s ease-in-out infinite; }
@keyframes pulse-glow {
 0%, 100% { box-shadow: 0 6px 18px rgba(0,0,0,.5), 0 0 var(--rest-glow) var(--glow, transparent), 0 0 0 1px rgba(255,255,255,.05) inset; }
 50% { box-shadow: 0 6px 18px rgba(0,0,0,.5), 0 0 calc(var(--rest-glow) * 1.6) var(--glow, transparent), 0 0 0 1px rgba(255,255,255,.08) inset; }
}
.card--r1, .card--r2, .card--r3, .card--r4, .card--r5, .card--r6 { z-index: 2; }
.card--r3:hover, .card--r3.live, .card--r4:hover, .card--r4.live,
.card--r5:hover, .card--r5.live, .card--r6:hover, .card--r6.live { z-index: 40; }
/* ---------------------------------------------------------------------
 Foil + glare, ported from the real simeydotme/pokemon-cards-css source
 (base.css + regular-holo.css + reverse-holo.css + v-regular.css), with
 our category hue(s) standing in for their fixed rainbow/sunpillar colors.
 Two details that earlier attempts got wrong and that matter a lot:
 1. contrast() does the real work β€” a smooth gradient blended color-dodge
 does very little; brightness/contrast/saturate crunch it into foil.
 2. background-position for the rainbow layer is NOT the raw pointer
 position β€” it's the pointer compressed into a narrow 37-63%/33-67%
 band (--bgx/--bgy, set in JS) and then amplified 2.6x/3.5x. Feeding
 the raw 0-100 pointer into that amplification (an earlier version of
 this page did) makes it swing wildly instead of panning smoothly.
 Raw pointer (--px/--py) is still used, unscaled, for the glare/vignette
 center β€” exactly like their --pointer-x/--pointer-y.
 --------------------------------------------------------------------- */
.card__foil {
 position: absolute; inset: 0; border-radius: inherit; z-index: 4;
 pointer-events: none; mix-blend-mode: color-dodge;
 opacity: calc(var(--op, 0) * var(--foil-op, 1));
 transition: opacity .4s ease;
 /* regular-holo: a 5-stop hue wheel (72deg apart) repeated 3x β€” an evenly
 spaced repeating-linear-gradient auto-spaces stops when none are given,
 same as their 5 fixed colors x3 β€” plus a 1px scanline texture blended in */
 background-image:
 repeating-linear-gradient(110deg in oklch,
 hsl(var(--hue), 90%, 60%), hsl(calc(var(--hue) + 72), 90%, 60%),
 hsl(calc(var(--hue) + 144), 90%, 60%), hsl(calc(var(--hue) + 216), 90%, 60%),
 hsl(calc(var(--hue) + 288), 90%, 60%), hsl(var(--hue), 90%, 60%),
 hsl(calc(var(--hue) + 72), 90%, 60%), hsl(calc(var(--hue) + 144), 90%, 60%),
 hsl(calc(var(--hue) + 216), 90%, 60%), hsl(calc(var(--hue) + 288), 90%, 60%),
 hsl(var(--hue), 90%, 60%), hsl(calc(var(--hue) + 72), 90%, 60%),
 hsl(calc(var(--hue) + 144), 90%, 60%), hsl(calc(var(--hue) + 216), 90%, 60%),
 hsl(calc(var(--hue) + 288), 90%, 60%)
 ),
 repeating-linear-gradient(90deg, black 0, black 1px, #666 1px, #666 2px);
 background-position:
 calc(((50% - (var(--bgx, 50) * 1%)) * 2.6) + 50%) calc(((50% - (var(--bgy, 50) * 1%)) * 3.5) + 50%),
 center center;
 background-size: 400% 400%, cover;
 background-blend-mode: overlay;
 filter: brightness(1.1) contrast(1.1) saturate(1.2);
}
/* the "bars" pseudo-layer β€” a second, cross-hatched light/dark stripe
 pattern blended hard-light, offset from the rainbow via a different
 position formula so it reads as a separate foil layer catching the light */
.card__foil::before {
 content: ""; position: absolute; inset: 0; mix-blend-mode: hard-light;
 background-image:
 repeating-linear-gradient(90deg, black 6%, hsl(0,0%,70%) 9%, black 10.5%, hsl(0,0%,70%) 12%, black 15%, black 42%),
 repeating-linear-gradient(90deg, black 6%, hsl(0,0%,70%) 9%, black 10.5%, hsl(0,0%,70%) 12%, black 15%, black 30%);
 background-size: 200% 200%, 200% 200%;
 background-position:
 calc((((50% - (var(--bgx, 50) * 1%)) * 1.65) + 50%) + ((var(--bgy, 50) * 1%) * 0.5)) calc(var(--bgx, 50) * 1%),
 calc((((50% - (var(--bgx, 50) * 1%)) * -0.9) + 50%) - ((var(--bgy, 50) * 1%) * 0.75)) calc(var(--bgy, 50) * 1%);
 background-blend-mode: screen;
 filter: brightness(1.15) contrast(1.1);
}
/* the vignette pseudo-layer β€” darkens away from the cursor, blended
 luminosity at high contrast, giving the rainbow a "beam" shape */
.card__foil::after {
 content: ""; position: absolute; inset: 0; mix-blend-mode: luminosity;
 background-image: radial-gradient(farthest-corner circle at calc(var(--px, 50) * 1%) calc(var(--py, 50) * 1%),
 hsla(0, 0%, 90%, .8) 0%, hsla(0, 0%, 78%, .1) 25%, hsl(0, 0%, 0%) 90%);
 filter: brightness(.6) contrast(4);
}
/* Uncommon (1): reverse-holo.css's simpler style β€” no image asset available
 for --foil so it collapses to just the two grayscale gradients, i.e. a
 satin sheen with no rainbow color at all. No bars/vignette pseudos. */
.card--r1 .card__foil {
 background-image:
 radial-gradient(circle at calc(var(--px, 50) * 1%) calc(var(--py, 50) * 1%), #fff 5%, #000 50%, #fff 80%),
 linear-gradient(-45deg, #000 15%, #fff, #000 85%);
 background-blend-mode: soft-light, difference;
 background-size: 120% 120%, 200% 200%;
 background-position: center center, calc(var(--px, 50) * 1%) calc(var(--py, 50) * 1%);
 filter: brightness(.6) contrast(1.5) saturate(1);
}
.card--r1 .card__foil::before, .card--r1 .card__foil::after { content: none; }
/* Glare: base.css's white-to-black vignette, present on every rarity;
 regular-holo.css tightens it further (higher contrast, brighter core)
 for tier 2+ via the ::after pseudo below. */
.card__glare {
 position: absolute; inset: 0; border-radius: inherit; z-index: 5;
 pointer-events: none; mix-blend-mode: overlay;
 opacity: calc(var(--op, 0) * var(--glare-op, 1));
 transition: opacity .4s ease;
 background-image: radial-gradient(farthest-corner circle at calc(var(--px, 50) * 1%) calc(var(--py, 50) * 1%),
 hsla(0, 0%, 100%, .8) 10%, hsla(0, 0%, 100%, .65) 20%, hsla(0, 0%, 0%, .5) 90%);
 filter: brightness(.8) contrast(1.5);
}
.card__glare::after {
 content: ""; position: absolute; inset: 0; mix-blend-mode: overlay;
 background-image: radial-gradient(farthest-corner circle at calc(var(--px, 50) * 1%) calc(var(--py, 50) * 1%),
 hsl(180, 100%, 95%) 5%, hsla(0, 0%, 39%, .25) 55%, hsla(0, 0%, 0%, .36) 110%);
 filter: brightness(.6) contrast(3);
}
.card--r1 .card__glare { filter: brightness(.7) contrast(1.5); }
.card--r1 .card__glare::after { content: none; }
/* Rainbow Rare (5) and Secret Rare (6): the border-box gradient itself
 becomes an animated, cycling rainbow instead of the two category colors
 (set on .card__body below, alongside the regular two-color version). */
.card--r5 .card__body, .card--r6 .card__body {
 /* var(--card-bg) itself expands to TWO comma-separated background images
 (the vignette + the diagonal blend), so this shorthand actually has
 FOUR layers total, not three β€” sheen, vignette, diagonal, rainbow.
 background-size/-position lists cycle when short of the layer count,
 so a 3-value list here silently mis-assigns 320% 100% to the diagonal
 layer (cutting our smooth blend off partway through) and leaves the
 rainbow layer unstretched. Every list below is spelled out to 4 values
 so nothing shifts onto the wrong layer. */
 background:
 linear-gradient(150deg, rgba(255,255,255,.10), transparent 40%) padding-box,
 var(--card-bg) padding-box,
 linear-gradient(120deg in oklch, #ff7a7a, #ffcf7a, #b9ff7a, #7affce, #7ac8ff, #c58bff, #ff7ad4, #ff7a7a) border-box;
 background-size: auto, auto, auto, 320% 100%;
 background-position: 0 0, 0 0, 0 0, 0 0;
 animation: rainbow-border 6s linear infinite;
}
/* Secret Rare (6) β€” the one card every major lab uses gets the "full art V"
 treatment from v-regular.css: two mirrored band layers (one blended via
 `hue`, one via `hard-light`, moving in opposite directions as the pointer
 moves β€” the "shifts opposite ways when tilted" look) plus a grain texture,
 instead of the shared repeating-rainbow foil every other tier uses. */
.card--r6 .card__foil::before { content: none; }
.card--r6 .card__foil {
 background-image:
 var(--grain),
 repeating-linear-gradient(0deg in oklch,
 hsl(var(--hue), 90%, 65%) 5%, hsl(calc(var(--hue) + 60), 90%, 65%) 10%,
 hsl(calc(var(--hue) + 120), 90%, 65%) 15%, hsl(var(--hue2), 90%, 65%) 20%,
 hsl(calc(var(--hue2) + 60), 90%, 65%) 25%, hsl(calc(var(--hue2) + 120), 90%, 65%) 30%,
 hsl(var(--hue), 90%, 65%) 35%
 ),
 repeating-linear-gradient(133deg,
 #0e152e 0%, hsl(180, 10%, 60%) 3.8%, hsl(180, 29%, 66%) 4.5%,
 hsl(180, 10%, 60%) 5.2%, #0e152e 10%, #0e152e 12%
 );
 background-blend-mode: screen, hue, hard-light;
 background-size: 500px 100%, 200% 700%, 300% 100%;
 background-position:
 center,
 0% calc(var(--bgy, 50) * 1%),
 calc(var(--bgx, 50) * 1%) calc(var(--bgy, 50) * 1%);
 /* toned down from contrast(2.4) β€” at full opacity that washed the card's
 own text out on hover; the reduced --foil-op above does most of the
 work, this keeps what does show from being harsh on top of that */
 filter: brightness(.9) contrast(1.6) saturate(.8);
}
.card--r6 .card__foil::after {
 mix-blend-mode: soft-light;
 background-image: var(--grain);
 background-position: 0% calc(var(--bgy, 50) * 1%);
 background-size: 500px 100%;
 filter: brightness(1) contrast(1.4) saturate(1.2);
}
.card--r6 .card__glare::after { content: none; }
.card--r6 .card__glare {
 background-image: radial-gradient(
 farthest-corner circle at calc(var(--px, 50) * 1%) calc(var(--py, 50) * 1%),
 hsl(0, 0%, 100%) 0%, hsla(210, 3%, 60%, .35) 45%, hsla(0, 0%, 20%, .85) 130%
 );
 mix-blend-mode: hard-light;
 filter: brightness(.95) contrast(1.3);
}
@keyframes rainbow-border { to { background-position: 0 0, 0 0, 0 0, 320% 0; } }
/* content */
.card__body {
 position: relative; z-index: 2; height: 100%; border-radius: inherit;
 padding: 16px 16px 14px; display: flex; flex-direction: column;
 border: var(--border-w) solid transparent;
 /* border-box gradient border trick: content layers paint over the padding
 box, the last layer (a 2-color gradient) shows only in the border ring.
 --accent3 defaults to --accent2 for single-category cards, so this is a
 solid-looking border there and a genuine two-color gradient for dual. */
 background:
 linear-gradient(150deg, rgba(255,255,255,.10), transparent 40%) padding-box,
 var(--card-bg) padding-box,
 linear-gradient(135deg in oklch, var(--accent2), var(--accent3)) border-box;
 overflow: hidden;
}
.card__badge {
 align-self: flex-start; font-size: 10px; letter-spacing: .06em; font-weight: 700;
 text-transform: uppercase; color: #0d0e12;
 background: linear-gradient(100deg in oklch, var(--accent2), var(--accent3));
 border-radius: 999px; padding: 3px 9px; margin-bottom: 10px;
}
.card__rarity {
 align-self: flex-start; font-size: 9.5px; font-weight: 800; letter-spacing: .05em;
 text-transform: uppercase; margin: -4px 0 8px; color: rgba(255,255,255,.6);
}
.card--r3 .card__rarity, .card--r4 .card__rarity { color: #fff; text-shadow: 0 0 6px var(--glow); }
.card--r5 .card__rarity, .card--r6 .card__rarity {
 background: linear-gradient(100deg in oklch, #ff9a7a, #ffe27a, #9aff9a, #7ad4ff, #d59aff);
 background-size: 260% 100%; animation: hue-drift 4s linear infinite;
 -webkit-background-clip: text; background-clip: text; color: transparent;
}
.card__title {
 font-size: 16px; font-weight: 700; line-height: 1.25; margin: 0 0 auto;
 color: #fff; text-shadow: 0 1px 3px rgba(0,0,0,.5);
}
.card__mid { flex: 1; }
.card__stat {
 margin-top: 10px; font-size: 12.5px; color: rgba(255,255,255,.85);
 display: flex; align-items: baseline; gap: 5px;
}
.card__stat b { font-size: 22px; color: #fff; }
.card__desc {
 font-size: 11px; line-height: 1.5; color: rgba(255,255,255,.72);
 margin-top: 8px; max-height: 62px; overflow: hidden;
 display: -webkit-box; -webkit-line-clamp: 4; -webkit-box-orient: vertical;
}
.card__foot {
 margin-top: 10px; display: flex; gap: 6px; flex-wrap: wrap; position: relative; z-index: 6;
}
.card__foot a {
 font-size: 10.5px; color: #0d0e12; background: rgba(255,255,255,.85);
 padding: 4px 9px; border-radius: 999px; text-decoration: none; font-weight: 600;
 transition: background .15s;
}
.card__foot a:hover { background: #fff; }
.card__foot .none {
 font-size: 10.5px; color: rgba(255,255,255,.45); padding: 4px 0;
}
.card__usedby {
 margin-top: 9px; display: flex; align-items: center; gap: 6px;
 position: relative; z-index: 6;
}
.card__usedby-label {
 font-size: 9.5px; letter-spacing: .04em; text-transform: uppercase;
 color: rgba(255,255,255,.55); font-weight: 700;
}
.lab-icon {
 display: inline-flex; width: 16px; height: 16px; color: rgba(255,255,255,.92);
 filter: drop-shadow(0 1px 2px rgba(0,0,0,.5));
}
.lab-icon svg { width: 100%; height: 100%; }
/* per-category resting gradient + accent (set inline per card via CSS vars) */
.card {
 /* a clean diagonal 50/50 split between the two category colors (collapses
 to one flat color when --accent3 == --accent2), with a soft vignette
 layered on top for depth so the corners still fall off to near-black */
 /* a single, genuinely continuous 2-stop diagonal blend β€” no plateau
 stops. A gradient with flat color at 0-22% then a sudden ramp to
 78-100% keeps the same color at each point (no jump) but the RATE of
 change kinks abruptly at those stops, which reads as a hard seam once
 the vignette layers on top. A plain 2-color ramp has no such kink. */
 /* "in oklch" changes how the browser interpolates BETWEEN the two color
 stops. Default (sRGB) interpolation between very different hues (e.g.
 green -> amber) dips through a muddy gray/brown in the middle β€” that
 dip is the "abrupt band" complaint, not a plateau-stop bug. oklch
 interpolation stays perceptually smooth the whole way across. */
 --card-bg:
 radial-gradient(130% 130% at 50% 0%, transparent 35%, rgba(0,0,0,.55) 100%),
 linear-gradient(
 135deg in oklch,
 color-mix(in srgb, var(--accent2) 72%, #14151b) 0%,
 color-mix(in srgb, var(--accent3) 72%, #14151b) 100%
 );
 --glow: color-mix(in srgb, color-mix(in srgb, var(--accent2) 50%, var(--accent3) 50%) 40%, transparent);
}
.hidden { display: none !important; }
</style>
</head>
<body>
<div class="hero">
 <h1>Benchmark Holo Deck</h1>
 <p>Every actively-used benchmark from
 <a href="https://huggingface.co/datasets/SaylorTwift/llm-benchmark-usage" target="_blank">SaylorTwift/llm-benchmark-usage</a>,
 rendered as a card colored by category. Rarity β€” Common, Uncommon, Rare, Rare Holo, Ultra Rare,
 Rainbow Rare, up to a one-of-a-kind Secret Rare β€” is set by how many major labs' <em>latest</em>
 release (OpenAI, Anthropic, Kimi, Z.ai, DeepSeek, Qwen, Meta) report that benchmark: the more labs, the
 stronger the foil and glow, and the higher it floats in the deck. The "Used by" row shows
 which labs those are, and a card links to a "πŸ† Leaderboard" instead of a plain HF dataset
 when its dataset is officially registered on the Hub. Move your mouse over a card to catch
 the shine β€” tap/hold on mobile.</p>
 <p class="note">Showing benchmarks used by 3+ models, with at least one report in the last 6 months β€”
 <span id="hidden-note"></span></p>
</div>
<div class="controls">
 <div class="pillrow" id="cat-pills"></div>
 <div class="spacer"></div>
 <input class="search" id="search" type="search" placeholder="Search benchmark…">
 <select class="sortsel" id="sortsel">
 <option value="models">Sort: Most used</option>
 <option value="labs">Sort: Used by (major labs)</option>
 <option value="name">Sort: Name (A–Z)</option>
 <option value="category">Sort: Category</option>
 </select>
</div>
<div class="count-note" id="count-note"></div>
<div class="grid" id="grid"></div>
<script id="lab-icons" type="application/json">{"icons": {"openai": "<svg fill=\"currentColor\" fill-rule=\"evenodd\" height=\"1em\" style=\"flex:none;line-height:1\" viewBox=\"0 0 24 24\" width=\"1em\" xmlns=\"http://www.w3.org/2000/svg\"><title>OpenAI</title><path d=\"M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z\"></path></svg>", "anthropic": "<svg fill=\"currentColor\" fill-rule=\"evenodd\" height=\"1em\" style=\"flex:none;line-height:1\" viewBox=\"0 0 24 24\" width=\"1em\" xmlns=\"http://www.w3.org/2000/svg\"><title>Anthropic</title><path d=\"M13.827 3.52h3.603L24 20h-3.603l-6.57-16.48zm-7.258 0h3.767L16.906 20h-3.674l-1.343-3.461H5.017l-1.344 3.46H0L6.57 3.522zm4.132 9.959L8.453 7.687 6.205 13.48H10.7z\"></path></svg>", "kimi": "<svg fill=\"currentColor\" fill-rule=\"evenodd\" height=\"1em\" style=\"flex:none;line-height:1\" viewBox=\"0 0 24 24\" width=\"1em\" xmlns=\"http://www.w3.org/2000/svg\"><title>Kimi</title><path d=\"M21.846 0a1.923 1.923 0 110 3.846H20.15a.226.226 0 01-.227-.226V1.923C19.923.861 20.784 0 21.846 0z\"></path><path d=\"M11.065 11.199l7.257-7.2c.137-.136.06-.41-.116-.41H14.3a.164.164 0 00-.117.051l-7.82 7.756c-.122.12-.302.013-.302-.179V3.82c0-.127-.083-.23-.185-.23H3.186c-.103 0-.186.103-.186.23V19.77c0 .128.083.23.186.23h2.69c.103 0 .186-.102.186-.23v-3.25c0-.069.025-.135.069-.178l2.424-2.406a.158.158 0 01.205-.023l6.484 4.772a7.677 7.677 0 003.453 1.283c.108.012.2-.095.2-.23v-3.06c0-.117-.07-.212-.164-.227a5.028 5.028 0 01-2.027-.807l-5.613-4.064c-.117-.078-.132-.279-.028-.381z\"></path></svg>", "deepseek": "<svg fill=\"currentColor\" fill-rule=\"evenodd\" height=\"1em\" style=\"flex:none;line-height:1\" viewBox=\"0 0 24 24\" width=\"1em\" xmlns=\"http://www.w3.org/2000/svg\"><title>DeepSeek</title><path d=\"M23.748 4.482c-.254-.124-.364.113-.512.234-.051.039-.094.09-.137.136-.372.397-.806.657-1.373.626-.829-.046-1.537.214-2.163.848-.133-.782-.575-1.248-1.247-1.548-.352-.156-.708-.311-.955-.65-.172-.241-.219-.51-.305-.774-.055-.16-.11-.323-.293-.35-.2-.031-.278.136-.356.276-.313.572-.434 1.202-.422 1.84.027 1.436.633 2.58 1.838 3.393.137.093.172.187.129.323-.082.28-.18.552-.266.833-.055.179-.137.217-.329.14a5.526 5.526 0 01-1.736-1.18c-.857-.828-1.631-1.742-2.597-2.458a11.365 11.365 0 00-.689-.471c-.985-.957.13-1.743.388-1.836.27-.098.093-.432-.779-.428-.872.004-1.67.295-2.687.684a3.055 3.055 0 01-.465.137 9.597 9.597 0 00-2.883-.102c-1.885.21-3.39 1.102-4.497 2.623C.082 8.606-.231 10.684.152 12.85c.403 2.284 1.569 4.175 3.36 5.653 1.858 1.533 3.997 2.284 6.438 2.14 1.482-.085 3.133-.284 4.994-1.86.47.234.962.327 1.78.397.63.059 1.236-.03 1.705-.128.735-.156.684-.837.419-.961-2.155-1.004-1.682-.595-2.113-.926 1.096-1.296 2.746-2.642 3.392-7.003.05-.347.007-.565 0-.845-.004-.17.035-.237.23-.256a4.173 4.173 0 001.545-.475c1.396-.763 1.96-2.015 2.093-3.517.02-.23-.004-.467-.247-.588zM11.581 18c-2.089-1.642-3.102-2.183-3.52-2.16-.392.024-.321.471-.235.763.09.288.207.486.371.739.114.167.192.416-.113.603-.673.416-1.842-.14-1.897-.167-1.361-.802-2.5-1.86-3.301-3.307-.774-1.393-1.224-2.887-1.298-4.482-.02-.386.093-.522.477-.592a4.696 4.696 0 011.529-.039c2.132.312 3.946 1.265 5.468 2.774.868.86 1.525 1.887 2.202 2.891.72 1.066 1.494 2.082 2.48 2.914.348.292.625.514.891.677-.802.09-2.14.11-3.054-.614zm1-6.44a.306.306 0 01.415-.287.302.302 0 01.2.288.306.306 0 01-.31.307.303.303 0 01-.304-.308zm3.11 1.596c-.2.081-.399.151-.59.16a1.245 1.245 0 01-.798-.254c-.274-.23-.47-.358-.552-.758a1.73 1.73 0 01.016-.588c.07-.327-.008-.537-.239-.727-.187-.156-.426-.199-.688-.199a.559.559 0 01-.254-.078c-.11-.054-.2-.19-.114-.358.028-.054.16-.186.192-.21.356-.202.767-.136 1.146.016.352.144.618.408 1.001.782.391.451.462.576.685.914.176.265.336.537.445.848.067.195-.019.354-.25.452z\"></path></svg>", "zai": "<svg fill=\"currentColor\" fill-rule=\"evenodd\" height=\"1em\" style=\"flex:none;line-height:1\" viewBox=\"0 0 24 24\" width=\"1em\" xmlns=\"http://www.w3.org/2000/svg\"><title>Z.ai</title><path d=\"M12.105 2L9.927 4.953H.653L2.83 2h9.276zM23.254 19.048L21.078 22h-9.242l2.174-2.952h9.244zM24 2L9.264 22H0L14.736 2H24z\"></path></svg>", "qwen": "<svg fill=\"currentColor\" fill-rule=\"evenodd\" height=\"1em\" style=\"flex:none;line-height:1\" viewBox=\"0 0 24 24\" width=\"1em\" xmlns=\"http://www.w3.org/2000/svg\"><title>Qwen</title><path d=\"M12.604 1.34c.393.69.784 1.382 1.174 2.075a.18.18 0 00.157.091h5.552c.174 0 .322.11.446.327l1.454 2.57c.19.337.24.478.024.837-.26.43-.513.864-.76 1.3l-.367.658c-.106.196-.223.28-.04.512l2.652 4.637c.172.301.111.494-.043.77-.437.785-.882 1.564-1.335 2.34-.159.272-.352.375-.68.37-.777-.016-1.552-.01-2.327.016a.099.099 0 00-.081.05 575.097 575.097 0 01-2.705 4.74c-.169.293-.38.363-.725.364-.997.003-2.002.004-3.017.002a.537.537 0 01-.465-.271l-1.335-2.323a.09.09 0 00-.083-.049H4.982c-.285.03-.553-.001-.805-.092l-1.603-2.77a.543.543 0 01-.002-.54l1.207-2.12a.198.198 0 000-.197 550.951 550.951 0 01-1.875-3.272l-.79-1.395c-.16-.31-.173-.496.095-.965.465-.813.927-1.625 1.387-2.436.132-.234.304-.334.584-.335a338.3 338.3 0 012.589-.001.124.124 0 00.107-.063l2.806-4.895a.488.488 0 01.422-.246c.524-.001 1.053 0 1.583-.006L11.704 1c.341-.003.724.032.9.34zm-3.432.403a.06.06 0 00-.052.03L6.254 6.788a.157.157 0 01-.135.078H3.253c-.056 0-.07.025-.041.074l5.81 10.156c.025.042.013.062-.034.063l-2.795.015a.218.218 0 00-.2.116l-1.32 2.31c-.044.078-.021.118.068.118l5.716.008c.046 0 .08.02.104.061l1.403 2.454c.046.081.092.082.139 0l5.006-8.76.783-1.382a.055.055 0 01.096 0l1.424 2.53a.122.122 0 00.107.062l2.763-.02a.04.04 0 00.035-.02.041.041 0 000-.04l-2.9-5.086a.108.108 0 010-.113l.293-.507 1.12-1.977c.024-.041.012-.062-.035-.062H9.2c-.059 0-.073-.026-.043-.077l1.434-2.505a.107.107 0 000-.114L9.225 1.774a.06.06 0 00-.053-.031zm6.29 8.02c.046 0 .058.02.034.06l-.832 1.465-2.613 4.585a.056.056 0 01-.05.029.058.058 0 01-.05-.029L8.498 9.841c-.02-.034-.01-.052.028-.054l.216-.012 6.722-.012z\"></path></svg>", "meta": "<svg fill=\"currentColor\" fill-rule=\"evenodd\" height=\"1em\" style=\"flex:none;line-height:1\" viewBox=\"0 0 24 24\" width=\"1em\" xmlns=\"http://www.w3.org/2000/svg\"><title>Meta</title><path d=\"M6.897 4c1.915 0 3.516.932 5.43 3.376l.282-.373c.19-.246.383-.484.58-.71l.313-.35C14.588 4.788 15.792 4 17.225 4c1.273 0 2.469.557 3.491 1.516l.218.213c1.73 1.765 2.917 4.71 3.053 8.026l.011.392.002.25c0 1.501-.28 2.759-.818 3.7l-.14.23-.108.153c-.301.42-.664.758-1.086 1.009l-.265.142-.087.04a3.493 3.493 0 01-.302.118 4.117 4.117 0 01-1.33.208c-.524 0-.996-.067-1.438-.215-.614-.204-1.163-.56-1.726-1.116l-.227-.235c-.753-.812-1.534-1.976-2.493-3.586l-1.43-2.41-.544-.895-1.766 3.13-.343.592C7.597 19.156 6.227 20 4.356 20c-1.21 0-2.205-.42-2.936-1.182l-.168-.184c-.484-.573-.837-1.311-1.043-2.189l-.067-.32a8.69 8.69 0 01-.136-1.288L0 14.468c.002-.745.06-1.49.174-2.23l.1-.573c.298-1.53.828-2.958 1.536-4.157l.209-.34c1.177-1.83 2.789-3.053 4.615-3.16L6.897 4zm-.033 2.615l-.201.01c-.83.083-1.606.673-2.252 1.577l-.138.199-.01.018c-.67 1.017-1.185 2.378-1.456 3.845l-.004.022a12.591 12.591 0 00-.207 2.254l.002.188c.004.18.017.36.04.54l.043.291c.092.503.257.908.486 1.208l.117.137c.303.323.698.492 1.17.492 1.1 0 1.796-.676 3.696-3.641l2.175-3.4.454-.701-.139-.198C9.11 7.3 8.084 6.616 6.864 6.616zm10.196-.552l-.176.007c-.635.048-1.223.359-1.82.933l-.196.198c-.439.462-.887 1.064-1.367 1.807l.266.398c.18.274.362.56.55.858l.293.475 1.396 2.335.695 1.114c.583.926 1.03 1.6 1.408 2.082l.213.262c.282.326.529.54.777.673l.102.05c.227.1.457.138.718.138.176.002.35-.023.518-.073.338-.104.61-.32.813-.637l.095-.163.077-.162c.194-.459.29-1.06.29-1.785l-.006-.449c-.08-2.871-.938-5.372-2.2-6.798l-.176-.189c-.67-.683-1.444-1.074-2.27-1.074z\"></path></svg>"}, "labels": {"openai": "OpenAI", "anthropic": "Anthropic", "kimi": "Kimi (Moonshot AI)", "deepseek": "DeepSeek", "zai": "Z.ai", "qwen": "Qwen (Alibaba)", "meta": "Meta"}}</script>
<script id="data-blob" type="application/json">{"cards": [{"name": "SWE-bench Verified", "models": 68, "categories": ["coding", "agentic"], "primary": "coding", "secondary": "agentic", "last_seen": "2026-08-18", "hf_dataset": "SWE-bench/SWE-bench_Verified", "implementation_url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified", "paper_url": "https://openai.com/index/introducing-swe-bench-verified/", "description": "A human-validated subset of 500 real-world GitHub issue/PR pairs from the SWE-bench dataset, where a model must generate a code patch to a repository that resolves the issue and passes the associated hidden test suite; a high score means the model can autonomously localize and fix real software bugs in large codebases (OpenAI later noted saturation and some flawed test cases in the set).", "labs": ["anthropic", "meta"]}, {"name": "SWE-bench Pro", "models": 38, "categories": ["coding", "agentic"], "primary": "coding", "secondary": "agentic", "last_seen": "2026-08-18", "hf_dataset": "ScaleAI/SWE-bench_Pro", "implementation_url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro", "paper_url": "https://arxiv.org/abs/2509.16941", "description": "A harder, enterprise-grade successor to SWE-bench from Scale AI, containing 1,865 long-horizon real-world GitHub issues drawn from 41 actively maintained repositories (business apps, B2B services, dev tools) that often require multi-file patches taking a human engineer hours to days; agents are scored by whether their generated patch makes the repo's hidden test suite pass, and scores remain low (best model ~23% pass@1 at publication), so a high score reflects genuine long-horizon, real-world software engineering capability.", "labs": ["anthropic", "meta", "qwen"]}, {"name": "Terminal-Bench 2.0", "models": 39, "categories": ["coding", "agentic"], "primary": "coding", "secondary": "agentic", "last_seen": "2026-07-05", "hf_dataset": "harborframework/terminal-bench-2.0", "implementation_url": "https://huggingface.co/datasets/harborframework/terminal-bench-2.0", "paper_url": "https://www.tbench.ai/news/announcement-2-0", "description": "An agentic benchmark of 89 hard, heavily human/LM-verified tasks run inside real terminal/command-line environments (e.g. compiling code, configuring servers, training models), each with a unique containerized environment and automated tests; a high score means an AI agent can autonomously complete complex, realistic end-to-end terminal workflows (frontier agents score under 65%).", "labs": []}, {"name": "Terminal-Bench Hard", "models": 10, "categories": ["coding", "agentic"], "primary": "coding", "secondary": "agentic", "last_seen": "2026-06-05", "hf_dataset": null, "implementation_url": "https://github.com/laude-institute/terminal-bench", "paper_url": "https://artificialanalysis.ai/methodology/intelligence-benchmarking#terminal-bench-hard", "description": "Artificial Analysis's curated 'hard' subset (44 tasks) of the original Terminal-Bench agentic CLI benchmark, distinct from the newer Terminal-Bench 2.0/2.1; an agent must use a live terminal inside a Docker sandbox to complete realistic software-engineering, sysadmin, and game-playing tasks, graded pass/fail by hidden test suites. Has no dedicated paper of its own (methodology documented by Artificial Analysis) and was retired from their Intelligence Index in mid-2026 in favor of Terminal-Bench 2.1.", "labs": []}, {"name": "SciCode", "models": 29, "categories": ["knowledge", "coding"], "primary": "knowledge", "secondary": "coding", "last_seen": "2026-08-11", "hf_dataset": "SciCode1/SciCode", "implementation_url": "https://huggingface.co/datasets/SciCode1/SciCode", "paper_url": "https://arxiv.org/abs/2407.13168", "description": "SciCode tests whether a model can write correct Python code to solve real scientific-computing subproblems (338 subproblems drawn from 80 challenging main problems spanning physics, math, chemistry, biology, and materials science, taken from actual research papers/textbooks); a high score means the model can decompose and implement nontrivial scientific algorithms end-to-end, which is very hard β€” even top models solve well under 10% of full problems.", "labs": ["kimi", "meta"]}, {"name": "LiveCodeBench v6", "models": 36, "categories": ["coding"], "primary": "coding", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": "livecodebench/code_generation_lite", "implementation_url": "https://huggingface.co/datasets/livecodebench/code_generation_lite", "paper_url": "https://arxiv.org/abs/2403.07974", "description": "A contamination-resistant, continuously updated coding benchmark that scrapes new competitive-programming problems from LeetCode, AtCoder, and Codeforces tagged by contest date, then scores generated code by executing it against hidden test cases; 'v6' (version_tag release_v6) is the release window covering problems from May 2023 through April 2025 (1,055 problems), and a high score means correct, working code on genuinely unseen problems.", "labs": []}, {"name": "ARC-Challenge", "models": 31, "categories": ["commonsense"], "primary": "commonsense", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "allenai/ai2_arc", "implementation_url": "https://huggingface.co/datasets/allenai/ai2_arc", "paper_url": "https://arxiv.org/abs/1803.05457", "description": "The harder split of the AI2 Reasoning Challenge: grade-school-level multiple-choice science questions that were specifically selected because both a retrieval-based system and a word-co-occurrence baseline answered them incorrectly, making the set resistant to simple lexical-matching shortcuts; a high score indicates genuine scientific reasoning ability rather than surface-level retrieval or co-occurrence heuristics.", "labs": []}, {"name": "HellaSwag", "models": 37, "categories": ["commonsense"], "primary": "commonsense", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "Rowan/hellaswag", "implementation_url": "https://huggingface.co/datasets/Rowan/hellaswag", "paper_url": "https://arxiv.org/abs/1905.07830", "description": "A multiple-choice commonsense-inference benchmark where a model must pick, from four options produced via adversarial filtering, the most plausible continuation of a short everyday scenario or video-caption prefix; a high score indicates strong grounded commonsense reasoning about what happens next, since the distractor options are constructed to fool language models while remaining trivial (>95% accuracy) for humans.", "labs": []}, {"name": "MMLU", "models": 66, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "cais/mmlu", "implementation_url": "https://huggingface.co/datasets/cais/mmlu", "paper_url": "https://arxiv.org/abs/2009.03300", "description": "A multiple-choice question-answering benchmark spanning 57 academic and professional subjects (e.g., elementary math, US history, law, medicine, computer science); a high score indicates broad world knowledge and problem-solving ability across diverse domains, evaluated zero-/few-shot.", "labs": ["anthropic"]}, {"name": "WinoGrande", "models": 29, "categories": ["commonsense"], "primary": "commonsense", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "allenai/winogrande", "implementation_url": "https://huggingface.co/datasets/allenai/winogrande", "paper_url": "https://arxiv.org/abs/1907.10641", "description": "WinoGrande is a large-scale (~44k), adversarially-filtered collection of Winograd-schema-style fill-in-the-blank sentence pairs that test commonsense pronoun/coreference resolution; a high score indicates robust commonsense reasoning rather than exploitation of dataset-specific surface cues.", "labs": []}, {"name": "GSM8K", "models": 54, "categories": ["math"], "primary": "math", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "openai/gsm8k", "implementation_url": "https://huggingface.co/datasets/openai/gsm8k", "paper_url": "https://arxiv.org/abs/2110.14168", "description": "A set of ~8.5K grade-school math word problems that require 2-8 steps of elementary arithmetic reasoning to solve, with solutions written out in natural language; a high score indicates strong multi-step numerical/arithmetic reasoning.", "labs": []}, {"name": "GAIA", "models": 7, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-06-22", "hf_dataset": "gaia-benchmark/GAIA", "implementation_url": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "paper_url": "https://arxiv.org/abs/2311.12983", "description": "A benchmark of 466 real-world questions for general AI assistants, requiring multi-step reasoning, web browsing/tool use, and handling of multi-modal files (level 1-3 difficulty by how many tools/steps are needed); questions are designed to be easy for humans (~92% accuracy) but hard for AI systems, so a high score means an assistant can autonomously plan, invoke tools, and produce the single exact correct answer across diverse everyday tasks.", "labs": []}, {"name": "BrowseComp", "models": 48, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-08-18", "hf_dataset": "smolagents/browse_comp", "implementation_url": "https://github.com/openai/simple-evals", "paper_url": "https://arxiv.org/abs/2504.12516", "description": "A set of 1,266 adversarially constructed, hard-to-answer questions that require an AI agent to persistently search and browse the live web to uncover obscure, entangled facts; a high score reflects strong deep-research/web-browsing agent capability rather than raw parametric knowledge.", "labs": ["anthropic", "kimi"]}, {"name": "xBench-DeepSearch", "models": 8, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-06-22", "hf_dataset": "xbench/DeepSearch", "implementation_url": "https://huggingface.co/datasets/xbench/DeepSearch", "paper_url": "https://xbench.org/files/xbench_profession_v2.4.pdf", "description": "A Chinese-language, contamination-resistant agentic benchmark of 100 open-domain questions evaluating 'deep search': multi-step web retrieval, cross-source evidence gathering, and synthesis into a correct answer rather than single-hop fact lookup; a high score reflects an AI agent's ability to autonomously plan and execute long-horizon web research tasks mirroring real-world professional search/analysis workflows.", "labs": []}, {"name": "Seal-0", "models": 12, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-06-22", "hf_dataset": "vtllms/sealqa", "implementation_url": "https://huggingface.co/datasets/vtllms/sealqa", "paper_url": "https://arxiv.org/abs/2506.01062", "description": "The hardest 111-question core subset of SealQA, a search-augmented-QA benchmark of fact-seeking questions specifically curated so that web search returns conflicting, noisy, or unhelpful results (frontier models with browsing score near 0% on it, hence the name); a high score means a model can reason reliably over messy/adversarial real search results rather than being misled by them. This is the SealQA benchmark from a Virginia Tech-led paper; a connection to Scale AI's separate 'SEAL leaderboards' product (similar name) could not be verified.", "labs": []}, {"name": "HLE", "models": 71, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-08-18", "hf_dataset": "cais/hle", "implementation_url": "https://huggingface.co/datasets/cais/hle", "paper_url": "https://arxiv.org/abs/2501.14249", "description": "Humanity's Last Exam is a multi-modal benchmark of 2,500 expert-crowdsourced questions (multiple-choice and short-answer) spanning over a hundred subjects at the frontier of human knowledge, built to resist simple internet lookup; because current frontier models answer fewer than 10% correctly, a high score signals closed-ended academic capability at the edge of expert human performance.", "labs": ["anthropic", "deepseek", "kimi", "meta", "qwen"]}, {"name": "FrontierScience-Olympiad", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-02", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "FrontierScience-Research", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-02", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "LongBench v2", "models": 23, "categories": ["reasoning", "long_context"], "primary": "reasoning", "secondary": "long_context", "last_seen": "2026-08-08T01:50:52+00:00", "hf_dataset": "THUDM/LongBench-v2", "implementation_url": "https://huggingface.co/datasets/THUDM/LongBench-v2", "paper_url": "https://arxiv.org/abs/2412.15204", "description": "A long-context understanding benchmark of 503 challenging multiple-choice questions (contexts ranging 8k-2M words) covering single/multi-document QA, long in-context learning, long-dialogue understanding, code-repo understanding, and long structured-data understanding; a high score means the model can perform deep reasoning over realistic long contexts, not just retrieve isolated facts (human experts under time pressure only reach ~53.7% accuracy).", "labs": ["qwen"]}, {"name": "IFBench", "models": 37, "categories": ["instruction_following"], "primary": "instruction_following", "secondary": null, "last_seen": "2026-08-11", "hf_dataset": "allenai/IFBench_test", "implementation_url": "https://huggingface.co/datasets/allenai/IFBench_test", "paper_url": "https://arxiv.org/abs/2507.02833", "description": "An instruction-following benchmark from AI2/UW pairing 300 held-out prompts with 58 new, programmatically verifiable output-format constraints (e.g. structural/lexical rules) that models were not trained to satisfy, testing generalization to unseen constraints rather than memorized ones; it was built because models saturate the older IFEval, and a high score means reliably obeying novel precise formatting instructions.", "labs": ["meta", "qwen"]}, {"name": "IFEval", "models": 47, "categories": ["instruction_following"], "primary": "instruction_following", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": "google/IFEval", "implementation_url": "https://huggingface.co/datasets/google/IFEval", "paper_url": "https://arxiv.org/abs/2311.07911", "description": "A set of ~500 prompts, each embedding one or more of 25 types of precisely machine-verifiable instructions (e.g. 'write more than 400 words', 'use the word AI at least 3 times'); it measures how reliably a model follows exact, checkable formatting/content constraints, scored by heuristic verification rather than a judge model.", "labs": []}, {"name": "TAU2-Bench", "models": 36, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "Genteki/tau2-bench", "implementation_url": "https://github.com/sierra-research/tau2-bench", "paper_url": "https://arxiv.org/abs/2506.07982", "description": "Evaluates conversational LLM agents in simulated multi-turn customer-service dialogues across domains like retail, airline, and telecom, where β€” unlike the original tau-bench β€” both the agent and a simulated user can independently call tools against a shared, partially observable environment state (dual control, e.g. joint troubleshooting); a high score means the agent reliably completes user goals while following domain policy, measured consistently across repeated trials (pass^k). Lives primarily as a runnable framework rather than a static HF dataset.", "labs": ["anthropic"]}, {"name": "VITA-Bench", "models": 8, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-06-22", "hf_dataset": "meituan-longcat/VitaBench", "implementation_url": "https://huggingface.co/datasets/meituan-longcat/VitaBench", "paper_url": "https://arxiv.org/abs/2509.26490", "description": "VitaBench is a real-world LLM-agent benchmark covering 'Versatile Interactive Tasks' across food delivery, in-store consumption, and online travel services, using 66 tools to construct 100 cross-scenario and 300 single-scenario multi-turn, tool-use tasks; a high score means an agent can correctly complete complex, dynamic, multi-domain service tasks end-to-end β€” current state-of-the-art models succeed on under 30% of cross-scenario tasks. Distinct from the unrelated VITA multimodal-LLM model series of the same name.", "labs": []}, {"name": "MMLU-Pro", "models": 73, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-08-11", "hf_dataset": "TIGER-Lab/MMLU-Pro", "implementation_url": "https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro", "paper_url": "https://arxiv.org/abs/2406.01574", "description": "A harder, expanded successor to MMLU with roughly 12,000 multiple-choice questions across 14 subjects (math, science, law, engineering, etc.), each with 10 answer options instead of 4 and more reasoning-heavy questions; it reduces the ceiling effects and prompt-sensitivity of MMLU, so a high score reflects stronger multi-step reasoning combined with broad domain knowledge rather than knowledge alone.", "labs": []}, {"name": "GPQA-Diamond", "models": 72, "categories": ["reasoning"], "primary": "reasoning", "secondary": null, "last_seen": "2026-08-18", "hf_dataset": "Idavidrein/gpqa", "implementation_url": "https://huggingface.co/datasets/Idavidrein/gpqa", "paper_url": "https://arxiv.org/abs/2311.12022", "description": "The 198-question hardest/highest-consensus subset of GPQA, a multiple-choice Q&A dataset of graduate-level, expert-written questions in biology, physics, and chemistry designed to be 'Google-proof' (skilled non-experts with web access still score only ~34%); a high score indicates expert-level scientific reasoning that resists lookup-based shortcuts. Hosted as the 'gpqa_diamond' config of the main GPQA dataset repo, not a separate repo.", "labs": ["anthropic", "kimi", "meta", "qwen"]}, {"name": "IMO-AnswerBench", "models": 25, "categories": ["math"], "primary": "math", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": "OpenEvals/IMO-AnswerBench", "implementation_url": "https://imobench.github.io/", "paper_url": "https://arxiv.org/abs/2511.01846", "description": "IMO-AnswerBench is a 400-problem benchmark of Olympiad-level math problems (Algebra, Combinatorics, Geometry, Number Theory across difficulty tiers from pre-IMO to IMO-Hard), rewritten from past competitions to resist memorization, where only the final verifiable answer is graded rather than the full proof; it is one part of Google DeepMind's IMO-Bench suite for evaluating mathematical reasoning.", "labs": []}, {"name": "AIME 2025", "models": 43, "categories": ["math"], "primary": "math", "secondary": null, "last_seen": "2026-06-30", "hf_dataset": "MathArena/aime_2025", "implementation_url": "https://huggingface.co/datasets/MathArena/aime_2025", "paper_url": "https://maa.org/maa-invitational-competitions/", "description": "Not a research-benchmark paper but the actual 2025 American Invitational Mathematics Examination (AIME I and II, 15 short-answer problems each, integer answers 0-999) repurposed as an LLM math-reasoning eval; a high score means the model can solve competition-level math problems exactly, and since it's a real, dated exam it is also prized as a low-contamination benchmark for models trained before 2025.", "labs": ["anthropic"]}, {"name": "HMMT Nov 2025", "models": 15, "categories": ["math"], "primary": "math", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": "MathArena/hmmt_nov_2025", "implementation_url": "https://huggingface.co/datasets/MathArena/hmmt_nov_2025", "paper_url": "https://www.hmmt.org/www/archive/problems", "description": "The November 2025 sitting of the Harvard-MIT Mathematics Tournament, a competition-math exam (covering rounds like Team, General, Theme, and Guts) repurposed as an LLM evaluation of competition-level mathematical problem solving on problems unseen in training data; a high score means the model can solve difficult, proof/answer-based olympiad-style math problems correctly. Not a benchmark paper β€” it's a real exam.", "labs": []}, {"name": "AA-LCR", "models": 24, "categories": ["long_context"], "primary": "long_context", "secondary": null, "last_seen": "2026-08-11", "hf_dataset": "ArtificialAnalysis/AA-LCR", "implementation_url": "https://huggingface.co/datasets/ArtificialAnalysis/AA-LCR", "paper_url": "https://artificialanalysis.ai/articles/announcing-aa-lcr", "description": "Artificial Analysis Long Context Reasoning (AA-LCR) is a benchmark of 100 hard, text-based questions that require reasoning across multiple real-world documents (~100k input tokens spanning ~230 documents in categories like company reports, legal, and academia); it tests genuine multi-document inference (not simple retrieval like needle-in-a-haystack), where a high score means a model can synthesize dispersed information into a single unambiguous, verified answer.", "labs": ["kimi", "meta"]}, {"name": "MRCR", "models": 7, "categories": ["long_context"], "primary": "long_context", "secondary": null, "last_seen": "2026-04-22", "hf_dataset": "openai/mrcr", "implementation_url": "https://huggingface.co/datasets/openai/mrcr", "paper_url": "https://arxiv.org/abs/2403.05530", "description": "The original Multi-Round Co-reference Resolution long-context benchmark, first described in Google's Gemini 1.5 technical report, where a model must reproduce one specific prior response (the 'needle') out of several near-identical distractor requests/responses scattered across a long synthetic multi-turn conversation; a high score means the model can accurately disambiguate and recall a specific needle even at very long context lengths. Google never publicly released the exact v1 eval data; the linked HF dataset is OpenAI's open reproduction of this task, distinct from Google's own later 'MRCR v2'.", "labs": []}, {"name": "Needle-in-a-haystack", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-28", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "KMMLU-Pro", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-28", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "KoBALT", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-28", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "CLIcK", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-28", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "MMMLU", "models": 32, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-06-30", "hf_dataset": "openai/MMMLU", "implementation_url": "https://huggingface.co/datasets/openai/MMMLU", "paper_url": "https://arxiv.org/abs/2009.03300", "description": "MMMLU (Multilingual MMLU) is a professional human-translated version of the MMLU test set into 14 languages (Arabic, Bengali, German, Spanish, French, Hindi, Indonesian, Italian, Japanese, Korean, Portuguese, Swahili, Yoruba, Chinese), covering the same 57 subjects and four-option multiple-choice format as MMLU; a high score indicates strong general knowledge and reasoning that holds up outside English rather than being an artifact of English-only training. Has no dedicated paper of its own β€” released directly as a dataset by OpenAI in 2024 as a translation of MMLU.", "labs": ["anthropic"]}, {"name": "WMT24++", "models": 13, "categories": ["translation"], "primary": "translation", "secondary": null, "last_seen": "2026-06-03", "hf_dataset": "google/wmt24pp", "implementation_url": "https://huggingface.co/datasets/google/wmt24pp", "paper_url": "https://arxiv.org/abs/2502.12404", "description": "A machine-translation quality benchmark that extends WMT24 from 9 to 55 languages and dialects, pairing English source segments (news, literary, speech, and social-media domains) with human-written reference translations and post-edits; a high score means a model produces fluent, accurate translations across a very broad and typologically diverse set of target languages, not just a handful of high-resource ones.", "labs": []}, {"name": "GPQA", "models": 32, "categories": ["reasoning"], "primary": "reasoning", "secondary": null, "last_seen": "2026-06-03", "hf_dataset": "Idavidrein/gpqa", "implementation_url": "https://huggingface.co/datasets/Idavidrein/gpqa", "paper_url": "https://arxiv.org/abs/2311.12022", "description": "GPQA (Graduate-Level Google-Proof Q&A) is a set of 448 multiple-choice questions in biology, physics, and chemistry written and validated by domain PhD holders/candidates, split into main, extended, and diamond subsets of increasing quality filtering; the questions are hard enough that skilled non-expert humans with unrestricted web access score only ~34%, so a high model score signals genuine graduate-level expert reasoning rather than lookup ability.", "labs": []}, {"name": "Multi-IF", "models": 9, "categories": ["instruction_following"], "primary": "instruction_following", "secondary": null, "last_seen": "2026-03-31", "hf_dataset": "facebook/Multi-IF", "implementation_url": "https://huggingface.co/datasets/facebook/Multi-IF", "paper_url": "https://arxiv.org/abs/2410.15553", "description": "A multilingual, multi-turn instruction-following benchmark extending IFEval with 4,501 three-turn conversations, each available in English and 7 additional languages; it tests whether a model can satisfy multiple compounding, programmatically verifiable instructions as a conversation progresses, with accuracy typically dropping with each added turn and for non-Latin-script languages.", "labs": []}, {"name": "MATH-500", "models": 26, "categories": ["math"], "primary": "math", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "HuggingFaceH4/MATH-500", "implementation_url": "https://huggingface.co/datasets/HuggingFaceH4/MATH-500", "paper_url": "https://arxiv.org/abs/2305.20050", "description": "MATH-500 is a fixed 500-problem subset of the MATH competition-mathematics dataset (covering 7 subjects and 5 difficulty levels), sampled and popularized by OpenAI's \"Let's Verify Step by Step\" paper as a standard held-out evaluation set; a high score means accurate final-answer mathematical reasoning across varied topics and difficulties.", "labs": []}, {"name": "MGSM", "models": 32, "categories": ["math"], "primary": "math", "secondary": null, "last_seen": "2026-06-27", "hf_dataset": "juletxara/mgsm", "implementation_url": "https://huggingface.co/datasets/juletxara/mgsm", "paper_url": "https://arxiv.org/abs/2210.03057", "description": "MGSM (Multilingual Grade School Math) takes the same 250 grade-school arithmetic word problems from a subset of GSM8K and has them human-translated into 10 typologically diverse languages (Spanish, French, German, Russian, Chinese, Japanese, Thai, Swahili, Bengali, Telugu); a high score means a model can do multi-step chain-of-thought arithmetic reasoning consistently across languages, not just in English.", "labs": []}, {"name": "SWE-bench Multilingual", "models": 35, "categories": ["coding", "agentic"], "primary": "coding", "secondary": "agentic", "last_seen": "2026-08-18", "hf_dataset": "SWE-bench/SWE-bench_Multilingual", "implementation_url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Multilingual", "paper_url": "https://www.swebench.com/multilingual.html", "description": "Extends the original (Python-only) SWE-bench to 300 real-world GitHub issue/pull-request pairs drawn from repositories in 9 other programming languages (Ruby, Rust, PHP, Java, Go, C, JavaScript, TypeScript, C++); a model or agent is given the issue and repo and must produce a patch that passes the hidden unit tests, so a high score reflects real-world software-engineering ability that generalizes beyond Python codebases.", "labs": []}, {"name": "Multi-SWE-bench", "models": 6, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-09", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "LiveCodeBench", "models": 22, "categories": ["coding"], "primary": "coding", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "livecodebench/code_generation_lite", "implementation_url": "https://huggingface.co/datasets/livecodebench/code_generation_lite", "paper_url": "https://arxiv.org/abs/2403.07974", "description": "A continuously updated, contamination-resistant competitive-programming benchmark that scrapes new, date-tagged problems from LeetCode, AtCoder, and Codeforces, evaluating code generation plus related coding abilities (self-repair, test-output prediction, code execution); a high score reflects strong, up-to-date functional coding ability rather than memorization of older training data.", "labs": []}, {"name": "BrowseComp-ZH", "models": 18, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": "PALIN2018/BrowseComp-ZH", "implementation_url": "https://huggingface.co/datasets/PALIN2018/BrowseComp-ZH", "paper_url": "https://arxiv.org/abs/2504.19314", "description": "A hard-to-find-facts web-browsing benchmark for Chinese, consisting of 289 multi-hop trivia-style questions across 11 domains, each reverse-engineered from a short, objective, verifiable answer; a high score means an agent can effectively search, navigate, and reason over the Chinese web (which has different infrastructure, sources, and censorship constraints than the English web) to recover obscure facts.", "labs": []}, {"name": "WideSearch", "models": 16, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-08-18", "hf_dataset": "ByteDance-Seed/WideSearch", "implementation_url": "https://huggingface.co/datasets/ByteDance-Seed/WideSearch", "paper_url": "https://arxiv.org/abs/2508.07999", "description": "An agentic web-search benchmark of 200 real-user-style tasks (100 English, 100 Chinese) spanning 15+ domains, each requiring an agent to exhaustively collect many atomic facts via live web search (e.g., a metric for every company in a sector) and assemble them into a structured, verifiable table; a high score means the agent can perform reliable, large-scale breadth-first information gathering rather than single-fact lookup β€” current systems solve only ~5% of tasks.", "labs": ["qwen"]}, {"name": "NL2Repo", "models": 16, "categories": ["coding", "agentic"], "primary": "coding", "secondary": "agentic", "last_seen": "2026-08-18", "hf_dataset": "YTL-AI-Labs-Data-XP-1/NL2Repo", "implementation_url": "https://github.com/multimodal-art-projection/NL2RepoBench", "paper_url": "https://arxiv.org/abs/2512.12730", "description": "NL2Repo-Bench evaluates long-horizon, repository-scale code generation: given only a natural-language requirements document and an empty workspace, an agent must autonomously design, implement, and package a complete, installable multi-file Python library with no scaffolding or tests provided; scoring is execution-based against the original open-source project's upstream test suite, and even the best current agents average under 40.5% pass rate. No official HF dataset confirmed; released via the authors' GitHub project.", "labs": ["deepseek", "qwen", "zai"]}, {"name": "GDPval-AA", "models": 12, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-07-09", "hf_dataset": "openai/gdpval", "implementation_url": "https://huggingface.co/datasets/openai/gdpval", "paper_url": "https://arxiv.org/abs/2510.04374", "description": "GDPval is OpenAI's benchmark of 220 real-world, economically valuable professional tasks spanning 44 occupations across the top US GDP-contributing sectors (e.g. accounting, law, engineering, healthcare), where models produce real deliverables (documents, spreadsheets, slides) graded blindly against industry-expert work by human and automated graders; a high score means model outputs are approaching or matching experienced-professional quality. The '-AA' suffix denotes Artificial Analysis's own scoring/leaderboard harness built on this same OpenAI benchmark (distinct from the later, updated 'GDPval-AA v2'), not a separate benchmark or dataset.", "labs": ["anthropic"]}, {"name": "Toolathlon", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-02", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "BBH", "models": 36, "categories": ["reasoning"], "primary": "reasoning", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "lukaemon/bbh", "implementation_url": "https://huggingface.co/datasets/lukaemon/bbh", "paper_url": "https://arxiv.org/abs/2210.09261", "description": "BIG-Bench Hard is a curated suite of 23 tasks drawn from BIG-Bench on which prior language models failed to beat the average human rater, spanning algorithmic/symbolic reasoning (e.g. multi-step arithmetic, object counting, boolean expressions) and semantic reasoning (e.g. movie recommendation, causal judgment, date understanding), typically evaluated few-shot; a high score, especially when achieved via chain-of-thought prompting, indicates reasoning ability that matches or exceeds typical human performance on these previously hard tasks.", "labs": []}, {"name": "TriviaQA", "models": 24, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "mandarjoshi/trivia_qa", "implementation_url": "https://huggingface.co/datasets/mandarjoshi/trivia_qa", "paper_url": "https://arxiv.org/abs/1705.03551", "description": "TriviaQA (Joshi et al., 2017) is a reading-comprehension/open-domain QA dataset of over 650K question-answer-evidence triples, built from ~95K trivia question-answer pairs paired with independently gathered evidence documents (web and Wikipedia); it tests a model's ability to extract or recall answers from long, distantly-supervised evidence, with high scores reflecting strong reading comprehension and factual recall on compositional trivia questions.", "labs": []}, {"name": "MathVista", "models": 14, "categories": ["math"], "primary": "math", "secondary": null, "last_seen": "2026-06-02", "hf_dataset": "AI4Math/MathVista", "implementation_url": "https://huggingface.co/datasets/AI4Math/MathVista", "paper_url": "https://arxiv.org/abs/2310.02255", "description": "A benchmark aggregating 6,141 examples from 31 existing and newly-created datasets that combines mathematical reasoning (algebra, geometry, statistics, function analysis, etc.) with diverse visual contexts such as charts, plots, geometry diagrams, tables, and natural images; a high score reflects a foundation model's ability to jointly perceive visual/diagrammatic information and carry out correct mathematical reasoning over it.", "labs": []}, {"name": "HumanEval", "models": 33, "categories": ["coding"], "primary": "coding", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "openai/openai_humaneval", "implementation_url": "https://huggingface.co/datasets/openai/openai_humaneval", "paper_url": "https://arxiv.org/abs/2107.03374", "description": "OpenAI's original Python code-generation benchmark of 164 hand-written problems, each a function signature plus docstring that the model must complete; correctness is checked by running the completion against held-out unit tests (pass@k), so a high score means the model reliably synthesizes functionally correct code from a natural-language spec.", "labs": []}, {"name": "EvalPlus", "models": 18, "categories": ["coding"], "primary": "coding", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": null, "implementation_url": "https://github.com/evalplus/evalplus", "paper_url": "https://arxiv.org/abs/2305.01210", "description": "A code-synthesis evaluation framework/project (Liu, Xia, Wang, Zhang, 2023) that rigorously augments existing coding benchmarks with large numbers of automatically generated (LLM- and mutation-based) test cases to catch functionally incorrect code that weaker original test suites miss; its two flagship outputs are HumanEval+ and MBPP+, and when a single 'EvalPlus' or 'EvalPlus score' is reported (e.g. on the official leaderboard), it typically refers to the average pass@1 across HumanEval+ and MBPP+ under greedy decoding, rather than a separate dataset of its own.", "labs": []}, {"name": "BigCodeBench", "models": 7, "categories": ["coding"], "primary": "coding", "secondary": null, "last_seen": "2026-06-02", "hf_dataset": "bigcode/bigcodebench", "implementation_url": "https://huggingface.co/datasets/bigcode/bigcodebench", "paper_url": "https://arxiv.org/abs/2406.15877", "description": "A practical Python code-generation benchmark of 1,140 tasks requiring models to correctly compose diverse function calls across many libraries and domains (not just isolated algorithmic puzzles) to satisfy complex natural-language instructions, evaluated by executing generated code against hand-crafted test cases; a high score means a model can reliably use real-world libraries/APIs to complete non-trivial, tool-heavy programming tasks.", "labs": []}, {"name": "MultiPL-E", "models": 17, "categories": ["coding"], "primary": "coding", "secondary": null, "last_seen": "2026-06-02", "hf_dataset": "nuprl/MultiPL-E", "implementation_url": "https://huggingface.co/datasets/nuprl/MultiPL-E", "paper_url": "https://arxiv.org/abs/2208.08227", "description": "A system and benchmark that translates the test cases and prompts of Python code-generation benchmarks (HumanEval and MBPP) into 18+ other programming languages, evaluating a model via functional correctness (unit-test pass rate) of generated code; a high score means the model can generate correct, executable code broadly across many programming languages, not just Python.", "labs": []}, {"name": "AI2D", "models": 9, "categories": ["reasoning"], "primary": "reasoning", "secondary": null, "last_seen": "2026-06-02", "hf_dataset": "lmms-lab/ai2d", "implementation_url": "https://huggingface.co/datasets/lmms-lab/ai2d", "paper_url": "https://arxiv.org/abs/1603.07396", "description": "AI2 Diagrams: a visual question-answering benchmark of grade-school science diagrams (food webs, life cycles, cell structures) paired with multiple-choice questions about labeled diagram parts; a high score indicates strong diagram parsing and scientific visual reasoning.", "labs": []}, {"name": "ChartQA", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-02", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "MMMU", "models": 20, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-06-02", "hf_dataset": "MMMU/MMMU", "implementation_url": "https://huggingface.co/datasets/MMMU/MMMU", "paper_url": "https://arxiv.org/abs/2311.16502", "description": "Massive Multi-discipline Multimodal Understanding (MMMU) is a college-level benchmark of ~11.5K multimodal (image+text) questions drawn from exams, quizzes, and textbooks across 6 broad disciplines, 30 subjects, and 183 subfields, requiring joint perception of diverse image types (charts, diagrams, medical scans, music sheets, etc.) and expert-level domain reasoning; a high score indicates strong college/expert-level multimodal reasoning across many academic fields.", "labs": []}, {"name": "CharXiv", "models": 16, "categories": ["reasoning"], "primary": "reasoning", "secondary": null, "last_seen": "2026-07-14", "hf_dataset": "princeton-nlp/CharXiv", "implementation_url": "https://huggingface.co/datasets/princeton-nlp/CharXiv", "paper_url": "https://arxiv.org/abs/2406.18521", "description": "A chart-understanding benchmark built from 2,323 real, unfiltered charts drawn from arXiv papers, pairing each with descriptive questions (about basic chart elements) and reasoning questions (requiring synthesis across visual elements); a high score means a multimodal model can accurately read and reason over realistic, diverse scientific charts rather than the templated/synthetic charts that saturate older chart-QA benchmarks.", "labs": ["kimi"]}, {"name": "LongVideoBench", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "MLVU", "models": 10, "categories": ["long_context"], "primary": "long_context", "secondary": null, "last_seen": "2026-06-02", "hf_dataset": "MLVU/MVLU", "implementation_url": "https://huggingface.co/datasets/MLVU/MVLU", "paper_url": "https://arxiv.org/abs/2406.04264", "description": "Multi-task Long Video Understanding benchmark evaluating multimodal LLMs on videos ranging from 3 minutes to 2 hours (movies, surveillance, egocentric, cartoons, games) across 9 task types spanning holistic, single-detail, and multi-detail understanding; a high score indicates strong comprehension and reasoning over long-form video content, not just short clips.", "labs": []}, {"name": "MMVU", "models": 12, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": "yale-nlp/MMVU", "implementation_url": "https://huggingface.co/datasets/yale-nlp/MMVU", "paper_url": "https://arxiv.org/abs/2501.12380", "description": "A multi-discipline, expert-level video-understanding benchmark of 3,000 human-expert-annotated QA examples over 1,529 CC-licensed videos across 27 subjects in Science, Healthcare, Humanities & Social Sciences, and Engineering, each paired with expert reasoning rationales; a high score means a model applies genuine expert-level domain reasoning to video content rather than surface-level perception. From Yale NLP (CVPR 2025), a distinct project from Video-MMMU despite the similar naming.", "labs": ["kimi"]}, {"name": "Video-MME", "models": 20, "categories": ["perception"], "primary": "perception", "secondary": null, "last_seen": "2026-06-13", "hf_dataset": "lmms-lab/Video-MME", "implementation_url": "https://huggingface.co/datasets/lmms-lab/Video-MME", "paper_url": "https://arxiv.org/abs/2405.21075", "description": "Video-MME is the first comprehensive video-understanding benchmark for multimodal LLMs, comprising 900 videos (256 hours total, spanning short/medium/long durations from 11 seconds to 1 hour) across 6 domains and 30 subfields, paired with 2,700 manually annotated multiple-choice QA pairs and optional subtitle/audio inputs; a high score reflects robust video comprehension across varying temporal lengths and modalities.", "labs": ["kimi"]}, {"name": "RULER", "models": 17, "categories": ["long_context"], "primary": "long_context", "secondary": null, "last_seen": "2026-07-28", "hf_dataset": "simonjegou/ruler", "implementation_url": "https://github.com/NVIDIA/RULER", "paper_url": "https://arxiv.org/abs/2404.06654", "description": "A synthetic long-context benchmark from NVIDIA comprising 13 configurable tasks (needle-in-a-haystack retrieval variants, multi-hop tracing, aggregation, and long-context QA) evaluated at increasing sequence lengths; a high score means a model retains strong performance across diverse long-context tasks (not just simple retrieval) at its claimed context length, revealing its true 'effective' context size. No fixed HF dataset β€” NVIDIA ships it as a generation script producing synthetic examples on-the-fly.", "labs": []}, {"name": "SWE-bench", "models": 6, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-02", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "MultiChallenge", "models": 21, "categories": ["instruction_following"], "primary": "instruction_following", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": "ScaleAI/MultiChallenge", "implementation_url": "https://huggingface.co/datasets/ScaleAI/MultiChallenge", "paper_url": "https://arxiv.org/abs/2501.17399", "description": "A multi-turn conversational benchmark of 266 English dialogues that tests whether LLMs can hold up across four realistic challenge types β€” instruction retention, inference memory of earlier user-provided information, reliable versioned editing, and self-coherence β€” using an LLM-as-judge with instance-level rubrics; a high score means the model reliably tracks context and instructions across many turns rather than just in single-turn Q&A.", "labs": []}, {"name": "BFCL-V4", "models": 10, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-02-28", "hf_dataset": null, "implementation_url": "https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard", "paper_url": "https://proceedings.mlr.press/v267/patil25a.html", "description": "Version 4 of UC Berkeley's Gorilla Berkeley Function-Calling Leaderboard, testing whether an LLM selects the right function/tool, fills correct parameters, and avoids hallucinating undefined functions; v4 adds agentic evaluation scenarios (multi-hop web search with error recovery, agent memory management, format-sensitivity testing) on top of prior single-turn/multi-turn AST and execution-based categories. No v4-specific paper exists β€” the linked paper covers the BFCL framework through v3; v4 additions are documented only via Berkeley's blog posts.", "labs": []}, {"name": "MMLU-Redux", "models": 35, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-06-27", "hf_dataset": "edinburgh-dawg/mmlu-redux-2.0", "implementation_url": "https://huggingface.co/datasets/edinburgh-dawg/mmlu-redux-2.0", "paper_url": "https://arxiv.org/abs/2406.04127", "description": "A manually re-annotated correction of the MMLU benchmark that identifies and fixes ground-truth errors, ambiguous questions, and other annotation issues found in the original dataset (the current v2 release covers 5,700 re-annotated questions across all 57 MMLU subjects, up from an initial 3,000-question/30-subject release); a high score reflects genuine multitask knowledge and reasoning ability that isn't distorted by MMLU's original label noise.", "labs": []}, {"name": "C-Eval", "models": 35, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": "ceval/ceval-exam", "implementation_url": "https://huggingface.co/datasets/ceval/ceval-exam", "paper_url": "https://arxiv.org/abs/2305.08322", "description": "A Chinese-language, MMLU-style multiple-choice exam benchmark spanning 13,948 questions across 52 subjects (STEM, humanities, social science, and other professional/academic disciplines) at four difficulty levels; a high score indicates broad Chinese academic and professional knowledge plus reasoning ability.", "labs": []}, {"name": "SuperGPQA", "models": 23, "categories": ["reasoning"], "primary": "reasoning", "secondary": null, "last_seen": "2026-04-23", "hf_dataset": "m-a-p/SuperGPQA", "implementation_url": "https://huggingface.co/datasets/m-a-p/SuperGPQA", "paper_url": "https://arxiv.org/abs/2502.14739", "description": "A graduate-level, multiple-choice question-answering benchmark spanning 285 disciplines (STEM, humanities, social sciences, and applied/vocational fields) with 26,529 expert-vetted questions; a high score reflects broad, expert-level knowledge and reasoning far beyond the ~6 disciplines covered by the original GPQA.", "labs": []}, {"name": "HMMT Feb 2026", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-23", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "AIME 2026", "models": 17, "categories": ["math"], "primary": "math", "secondary": null, "last_seen": "2026-08-09", "hf_dataset": "MathArena/aime_2026", "implementation_url": "https://huggingface.co/datasets/MathArena/aime_2026", "paper_url": "https://artofproblemsolving.com/wiki/index.php/2026_AIME_I", "description": "The 2026 American Invitational Mathematics Examination (AIME I/II) repurposed as an LLM math-reasoning eval: 15 short-answer problems per exam (integer answers 0-999) covering advanced high-school competition mathematics; a high score means the model can solve difficult, multi-step olympiad-style math problems, typically measured via exact-match on the final integer answer. Not a benchmark paper β€” it's a real exam.", "labs": ["meta"]}, {"name": "MATH", "models": 39, "categories": ["math"], "primary": "math", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": "hendrycks/competition_math", "implementation_url": "https://huggingface.co/datasets/hendrycks/competition_math", "paper_url": "https://arxiv.org/abs/2103.03874", "description": "Tests free-response mathematical problem solving on 12,500 competition-level problems (from AMC, AIME, and similar competitions) spanning algebra, geometry, calculus, number theory, and more, each requiring a full step-by-step derivation ending in a final answer; a high score reflects strong multi-step symbolic/quantitative reasoning, and the benchmark is deliberately harder than grade-school-level datasets like GSM8K.", "labs": []}, {"name": "CRUXEval", "models": 10, "categories": ["coding", "reasoning"], "primary": "coding", "secondary": "reasoning", "last_seen": "2026-07-05", "hf_dataset": "cruxeval-org/cruxeval", "implementation_url": "https://huggingface.co/datasets/cruxeval-org/cruxeval", "paper_url": "https://arxiv.org/abs/2401.03065", "description": "Code Reasoning, Understanding, and eXecution Evaluation: 800 short Python functions each paired with an input-output example, split into CRUXEval-I (predict a valid input for a given output) and CRUXEval-O (predict the output for a given input), without executing the code; a high score indicates strong ability to mentally simulate program execution rather than just generate code from natural-language specs.", "labs": []}, {"name": "INCLUDE", "models": 19, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-04-06", "hf_dataset": "CohereLabs/include-base-44", "implementation_url": "https://huggingface.co/datasets/CohereLabs/include-base-44", "paper_url": "https://arxiv.org/abs/2411.19799", "description": "A multiple-choice knowledge/reasoning benchmark built from ~197K QA pairs sourced from real academic, professional, and occupational licensure exams across 44 languages, designed to test regional and cultural knowledge that generic translated benchmarks (like translated MMLU) miss; a high score indicates a model can reason correctly using knowledge specific to the local context in which each language is actually used, not just translated facts.", "labs": []}, {"name": "Arena-Hard", "models": 14, "categories": ["alignment"], "primary": "alignment", "secondary": null, "last_seen": "2026-04-06", "hf_dataset": "lmarena-ai/arena-hard-auto", "implementation_url": "https://huggingface.co/datasets/lmarena-ai/arena-hard-auto", "paper_url": "https://arxiv.org/abs/2406.11939", "description": "An automatic, pairwise LLM-evaluation benchmark (Arena-Hard-Auto) that uses several hundred challenging, diverse real-user prompts mined from Chatbot Arena/LMArena; a strong LLM judge compares each candidate model's responses head-to-head against a baseline model, and the resulting win rate is designed to closely approximate human preference rankings from live Chatbot Arena battles, so a high score means the model's overall response quality is judged to be strong on hard, realistic prompts.", "labs": []}, {"name": "Writing Bench", "models": 8, "categories": ["writing"], "primary": "writing", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": null, "implementation_url": "https://github.com/X-PLUG/WritingBench", "paper_url": "https://arxiv.org/abs/2503.05244", "description": "A text-only benchmark evaluating LLMs' generative writing ability across 6 domains and 100 subdomains (creative, persuasive, informative, technical writing), using query-dependent, LLM-judge-generated criteria plus a fine-tuned critic model rather than exact-match; a high score means the model's writing better satisfies instance-specific style, format, and content requirements.", "labs": []}, {"name": "AIME 2024", "models": 27, "categories": ["math"], "primary": "math", "secondary": null, "last_seen": "2026-06-21", "hf_dataset": "Maxwell-Jia/AIME_2024", "implementation_url": "https://huggingface.co/datasets/Maxwell-Jia/AIME_2024", "paper_url": "https://maa.org/maa-invitational-competitions/", "description": "The 2024 AIME (American Invitational Mathematics Examination) is a 15-question, integer-answer, competition-level high-school math exam (two sittings, AIME I and AIME II) repurposed as an LLM benchmark; a high score reflects strong performance on hard olympiad-style algebra, geometry, number-theory, and combinatorics problems. This is a real-world exam, not an academic benchmark paper.", "labs": []}, {"name": "BFCL v3", "models": 16, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-04-06", "hf_dataset": "gorilla-llm/Berkeley-Function-Calling-Leaderboard", "implementation_url": "https://huggingface.co/datasets/gorilla-llm/Berkeley-Function-Calling-Leaderboard", "paper_url": "https://gorilla.cs.berkeley.edu/blogs/13_bfcl_v3_multi_turn.html", "description": "BFCL v3 (Berkeley Function-Calling Leaderboard, version 3) tests whether an LLM can correctly select and invoke functions/tools given a user query and a set of function definitions, covering simple, parallel, multiple, and multi-turn/multi-step function-calling scenarios evaluated via AST matching and executable checks; a high score means the model reliably picks the right tool(s), fills in correct arguments, and handles multi-turn state changes across a call sequence.", "labs": []}, {"name": "Codeforces", "models": 23, "categories": ["coding"], "primary": "coding", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": "open-r1/codeforces", "implementation_url": "https://huggingface.co/datasets/open-r1/codeforces", "paper_url": "https://arxiv.org/abs/2203.07814", "description": "Uses real Codeforces competitive-programming contest problems, scored via the platform's Elo-style rating/percentile system, to gauge an LLM's algorithmic coding ability against the human competitor pool; a high Codeforces rating means performance comparable to strong human competitive programmers. Not a single purpose-built benchmark with one canonical paper β€” the practice of using live Codeforces contests for LLM evaluation was pioneered by DeepMind's AlphaCode paper and later popularized as a standard reported metric by OpenAI's o1/o3 'Competitive Programming with Large Reasoning Models' report.", "labs": []}, {"name": "PolyMATH", "models": 14, "categories": ["math"], "primary": "math", "secondary": null, "last_seen": "2026-02-28", "hf_dataset": "Qwen/PolyMath", "implementation_url": "https://huggingface.co/datasets/Qwen/PolyMath", "paper_url": "https://arxiv.org/abs/2504.18428", "description": "PolyMATH (\"Evaluating Mathematical Reasoning in Multilingual Contexts\") presents the same set of math word/competition problems, spanning four difficulty levels from elementary to olympiad/frontier level, professionally translated into 18 languages, to test whether a model's math reasoning ability is consistent across languages rather than just in English; a high score means strong, language-robust mathematical reasoning.", "labs": []}, {"name": "HMMT Feb 2025", "models": 14, "categories": ["math"], "primary": "math", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": "MathArena/hmmt_feb_2025", "implementation_url": "https://huggingface.co/datasets/MathArena/hmmt_feb_2025", "paper_url": "https://www.hmmt.org/www/archive/282", "description": "The February 2025 sitting of the Harvard-MIT Mathematics Tournament, a proof/short-answer competition-math exam for pre-college students, repurposed as an LLM benchmark by having models solve the same 30 hard olympiad-style problems (algebra, combinatorics, geometry, number theory); a high score indicates elite competition-level mathematical problem solving. Not a benchmark paper β€” it's a real exam.", "labs": []}, {"name": "MMLU-ProX", "models": 11, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-06-03", "hf_dataset": "li-lab/MMLU-ProX", "implementation_url": "https://huggingface.co/datasets/li-lab/MMLU-ProX", "paper_url": "https://arxiv.org/abs/2503.10497", "description": "A multilingual extension of MMLU-Pro: the same ~11,829 hard, multi-subject multiple-choice reasoning questions professionally translated via LLM-translation-plus-expert-review into 29 typologically diverse languages, enabling direct cross-lingual comparison; a high score means a model retains strong multi-subject reasoning across many languages rather than only in high-resource English.", "labs": []}, {"name": "NOVA-63", "models": 7, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-02-28", "hf_dataset": "zjy1298/NOVA-63", "implementation_url": "https://huggingface.co/datasets/zjy1298/NOVA-63", "paper_url": "https://www.researchgate.net/publication/397425551_NOVA-63_Native_Omni-lingual_Versatile_Assessments_of_63_Disciplines", "description": "Low confidence / could not fully verify. The name and an associated HF dataset (currently empty/'under review') suggest a multilingual, multi-discipline knowledge/reasoning evaluation, with '63' likely referring to 63 disciplines/categories rather than 63 languages; a matching-titled paper ('NOVA-63: Native Omni-lingual Versatile Assessments of 63 Disciplines') exists but its content, authors, and venue could not be confirmed. Treat this entry as unverified.", "labs": []}, {"name": "Global PIQA", "models": 7, "categories": ["commonsense"], "primary": "commonsense", "secondary": null, "last_seen": "2026-02-28", "hf_dataset": "mrlbenchmarks/global-piqa-nonparallel", "implementation_url": "https://huggingface.co/datasets/mrlbenchmarks/global-piqa-nonparallel", "paper_url": "https://arxiv.org/abs/2510.24081", "description": "A participatory, culturally-localized multilingual extension of PIQA physical commonsense reasoning, hand-built by 350+ researchers from 65+ countries and covering 100+ languages/136+ language varieties in multiple-choice format, with over half of items referencing local foods, customs, or other culturally specific content; a high score means a model picks the physically/practically sensible answer consistently across languages and cultures, not just in English.", "labs": []}, {"name": "MAXIFE", "models": 7, "categories": ["instruction_following"], "primary": "instruction_following", "secondary": null, "last_seen": "2026-02-28", "hf_dataset": null, "implementation_url": "https://github.com/OPPO-Mente-Lab/MaXIFE", "paper_url": "https://arxiv.org/abs/2506.01776", "description": "MaXIFE (Multilingual and Cross-lingual Instruction Following Evaluation, ACL 2025): a benchmark of 1,667 verifiable instruction-following tasks spanning 11 major/47 fine-grained instruction categories, evaluated monolingually across 23 languages and cross-lingually (English paired with 22 other languages); a high score means a model reliably satisfies precise formatting/content instructions both within and across languages.", "labs": []}, {"name": "MMMU-Pro", "models": 24, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": "MMMU/MMMU_Pro", "implementation_url": "https://huggingface.co/datasets/MMMU/MMMU_Pro", "paper_url": "https://arxiv.org/abs/2409.02813", "description": "A harder, more robust version of MMMU (Massive Multi-discipline Multimodal Understanding) that filters out questions answerable from text alone, expands the answer options, and adds a vision-only setting where the question itself is embedded in an image; a high score indicates genuine college-level multimodal (image+text) reasoning rather than text-only shortcuts.", "labs": ["kimi", "meta"]}, {"name": "MathVision", "models": 18, "categories": ["math"], "primary": "math", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": "MathLLMs/MathVision", "implementation_url": "https://huggingface.co/datasets/MathLLMs/MathVision", "paper_url": "https://arxiv.org/abs/2402.14804", "description": "MathVision (MATH-V) is a benchmark of 3,040 visual math problems drawn from real math competitions spanning 16 mathematical disciplines and 5 difficulty levels, requiring a model to jointly interpret diagrams/figures and perform mathematical reasoning to produce the correct answer; a high score indicates strong multimodal mathematical reasoning β€” even top models score far below human performance (~70%).", "labs": ["kimi"]}, {"name": "DynaMath", "models": 8, "categories": ["math"], "primary": "math", "secondary": null, "last_seen": "2026-04-21", "hf_dataset": "DynaMath/DynaMath_Sample", "implementation_url": "https://huggingface.co/datasets/DynaMath/DynaMath_Sample", "paper_url": "https://arxiv.org/abs/2411.00836", "description": "A dynamic visual math benchmark built from 501 seed questions (geometry, algebra, graph theory, statistics, scientific figures), each implemented as a Python program that procedurally generates multiple visual/numeric variants to probe multimodal LLMs' reasoning robustness; a high score means the model solves not just an original problem but stays consistently correct across randomized variations of it.", "labs": []}, {"name": "ZeroBench", "models": 15, "categories": ["reasoning"], "primary": "reasoning", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": "jonathan-roberts1/zerobench", "implementation_url": "https://huggingface.co/datasets/jonathan-roberts1/zerobench", "paper_url": "https://arxiv.org/abs/2502.09696", "description": "A set of 100 manually curated, highly difficult, multi-step visual reasoning questions (plus 334 sub-questions) covering diverse domains and image types, deliberately adversarially filtered so that current frontier large multimodal models score ~0% pass@1, giving the benchmark years of headroom for future progress; any non-trivial score reflects genuinely strong visual reasoning.", "labs": ["kimi"]}, {"name": "ZeroBench_sub", "models": 6, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-15", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "VlmsAreBlind", "models": 6, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-21", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "RealWorldQA", "models": 9, "categories": ["perception"], "primary": "perception", "secondary": null, "last_seen": "2026-04-21", "hf_dataset": "xai-org/RealworldQA", "implementation_url": "https://huggingface.co/datasets/xai-org/RealworldQA", "paper_url": "https://x.ai/blog/grok-1.5v", "description": "A visual question-answering benchmark of over 700 real-world photographs (many taken from inside vehicles) each paired with a question and verifiable answer; a high score indicates strong real-world spatial understanding and physical-scene reasoning. Originally released by xAI alongside Grok-1.5V.", "labs": []}, {"name": "MMStar", "models": 8, "categories": ["reasoning", "perception"], "primary": "reasoning", "secondary": "perception", "last_seen": "2026-04-21", "hf_dataset": "Lin-Chen/MMStar", "implementation_url": "https://huggingface.co/datasets/Lin-Chen/MMStar", "paper_url": "https://arxiv.org/abs/2403.20330", "description": "A vision-language benchmark of 1,500 multiple-choice questions across 6 core capabilities and 18 axes (coarse/fine perception, instance reasoning, logical reasoning, science, math), manually curated to keep only samples that genuinely require the image; a high score reflects genuine multimodal understanding rather than exploiting textual shortcuts or training-data leakage.", "labs": []}, {"name": "MMBench", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-21", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "SimpleVQA", "models": 11, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": "m-a-p/SimpleVQA", "implementation_url": "https://huggingface.co/datasets/m-a-p/SimpleVQA", "paper_url": "https://arxiv.org/abs/2502.13059", "description": "A multimodal factuality benchmark of 2,025 image-question-answer pairs across 9 task categories and 9 topics, where models answer short, objective natural-language questions about images that have static, verifiable reference answers; a high score indicates accurate, hallucination-free factual grounding in visual content.", "labs": []}, {"name": "HallusionBench", "models": 8, "categories": ["safety"], "primary": "safety", "secondary": null, "last_seen": "2026-04-15", "hf_dataset": "rayguan/HallusionBench", "implementation_url": "https://huggingface.co/datasets/rayguan/HallusionBench", "paper_url": "https://arxiv.org/abs/2310.14566", "description": "A diagnostic VQA benchmark (~1,100+ yes/no questions over ~455 image-question control sets, many with manually edited/paired images) that disentangles two multimodal LLM failure modes: language-prior-driven hallucination (ignoring the image) and visual illusion (misreading the image); a high score means the model correctly answers across both original and edited image variants rather than defaulting to language-prior guesses.", "labs": []}, {"name": "MMLongBench-Doc", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-02-28", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "CC-OCR", "models": 9, "categories": ["perception"], "primary": "perception", "secondary": null, "last_seen": "2026-04-21", "hf_dataset": "wulipc/CC-OCR", "implementation_url": "https://huggingface.co/datasets/wulipc/CC-OCR", "paper_url": "https://arxiv.org/abs/2412.02210", "description": "A comprehensive, challenging OCR benchmark from Alibaba/Qwen-affiliated researchers covering four tracks β€” multi-scene text reading, multilingual text reading, document parsing, and key information extraction β€” across 39 subsets and ~7,058 annotated images (including Chinese and English content); a high score indicates strong, generalizable OCR capability across scenes, languages, and document types.", "labs": []}, {"name": "OmniDocBench", "models": 19, "categories": ["perception"], "primary": "perception", "secondary": null, "last_seen": "2026-08-09", "hf_dataset": "opendatalab/OmniDocBench", "implementation_url": "https://huggingface.co/datasets/opendatalab/OmniDocBench", "paper_url": "https://arxiv.org/abs/2412.07626", "description": "A document-parsing/OCR benchmark of 981 real-world PDF pages spanning 9-10 diverse document types (academic papers, textbooks, financial reports, newspapers, handwritten notes, etc.), with fine-grained annotations for layout, text, tables, and formulas; a high score means a multimodal document-understanding pipeline can accurately extract structured content and reading order across varied, messy real-world layouts, not just clean scans.", "labs": ["kimi", "meta"]}, {"name": "OCRBench", "models": 10, "categories": ["perception"], "primary": "perception", "secondary": null, "last_seen": "2026-04-21", "hf_dataset": "echo840/OCRBench", "implementation_url": "https://huggingface.co/datasets/echo840/OCRBench", "paper_url": "https://arxiv.org/abs/2305.07895", "description": "A comprehensive benchmark for evaluating the text-recognition/OCR capability of large multimodal models, comprising roughly 1,000 image-text pairs across five task types (text recognition, scene-text VQA, document-oriented VQA, key information extraction, and handwritten mathematical expression recognition); a higher score reflects stronger understanding of text embedded in images across scenes, documents, handwriting, and formulas.", "labs": []}, {"name": "RefCOCO", "models": 9, "categories": ["perception"], "primary": "perception", "secondary": null, "last_seen": "2026-04-21", "hf_dataset": "lmms-lab/RefCOCO", "implementation_url": "https://huggingface.co/datasets/lmms-lab/RefCOCO", "paper_url": "https://arxiv.org/abs/1608.00272", "description": "A referring-expression-comprehension benchmark built on MS-COCO images, where given an image and a natural-language expression the model must localize the specific object/region being referred to; a high score means the model accurately grounds free-form referring expressions to the correct region, even among multiple similar objects.", "labs": []}, {"name": "CountBench", "models": 8, "categories": ["perception"], "primary": "perception", "secondary": null, "last_seen": "2026-04-21", "hf_dataset": "vikhyatk/CountBenchQA", "implementation_url": "https://huggingface.co/datasets/vikhyatk/CountBenchQA", "paper_url": "https://arxiv.org/abs/2302.12066", "description": "A text-image object-counting benchmark automatically curated from LAION-400M, containing images whose captions state a specific count (2-10) of a salient object, reformatted as VQA questions ('how many X are in the image?'); a high score means the vision-language model can accurately enumerate objects rather than guessing a plausible small number, a known weakness of VLMs.", "labs": []}, {"name": "ODinW13", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-15", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "ERQA", "models": 8, "categories": ["reasoning"], "primary": "reasoning", "secondary": null, "last_seen": "2026-04-21", "hf_dataset": "FlagEval/ERQA", "implementation_url": "https://github.com/embodiedreasoning/ERQA", "paper_url": "https://arxiv.org/abs/2503.20020", "description": "Embodied Reasoning Question Answering: a 400-question multiple-choice visual QA benchmark from Google DeepMind (introduced alongside Gemini Robotics) testing embodied/physical reasoning from images β€” spatial reasoning, trajectory reasoning, action reasoning, state estimation, pointing, multi-view correspondence, task reasoning; a high score means the model can reason about physical scenes the way an embodied agent would need to. No official HF dataset from Google/DeepMind was found; canonical source is GitHub.", "labs": []}, {"name": "EmbSpatialBench", "models": 9, "categories": ["perception"], "primary": "perception", "secondary": null, "last_seen": "2026-04-21", "hf_dataset": "Phineas476/EmbSpatial-Bench", "implementation_url": "https://huggingface.co/datasets/Phineas476/EmbSpatial-Bench", "paper_url": "https://arxiv.org/abs/2406.05756", "description": "An embodied spatial-reasoning benchmark of multiple-choice QA pairs automatically derived from 3D embodied scenes, testing understanding of six egocentric spatial relationships (closest/farthest, left/right, above/below) between objects relevant to robot navigation and manipulation; a high score indicates a vision-language model can correctly infer object spatial relations from a first-person viewpoint.", "labs": []}, {"name": "RefSpatialBench", "models": 9, "categories": ["reasoning"], "primary": "reasoning", "secondary": null, "last_seen": "2026-04-21", "hf_dataset": "BAAI/RefSpatial-Bench", "implementation_url": "https://huggingface.co/datasets/BAAI/RefSpatial-Bench", "paper_url": "https://arxiv.org/abs/2506.04308", "description": "A robotics-oriented spatial-referring benchmark (from the RoboRefer project) using real-world cluttered scene images where a model must output 2D point coordinates for a target object described via multi-step spatial reasoning (relative position, ordinal reference, or where to place an object); a high score reflects accurate grounding under complex, compositional spatial language.", "labs": []}, {"name": "Hypersim", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-02-28", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "SUNRGBD", "models": 6, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-02-28", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "Nuscenes", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-02-28", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "VideoMMMU", "models": 16, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": "lmms-lab/VideoMMMU", "implementation_url": "https://huggingface.co/datasets/lmms-lab/VideoMMMU", "paper_url": "https://arxiv.org/abs/2501.13826", "description": "A multi-discipline, multimodal benchmark of 300 expert-level educational videos (across Art, Business, Science, Medicine, Humanities, and Engineering) with 900 human-annotated questions structured into Perception, Comprehension, and Adaptation stages, measuring how well a model acquires and applies knowledge learned from watching video content; a high score, including a high Ξ”knowledge (post-video minus pre-video accuracy), indicates strong video-based knowledge acquisition and application.", "labs": []}, {"name": "MVBench", "models": 9, "categories": ["reasoning", "perception"], "primary": "reasoning", "secondary": "perception", "last_seen": "2026-04-21", "hf_dataset": "OpenGVLab/MVBench", "implementation_url": "https://huggingface.co/datasets/OpenGVLab/MVBench", "paper_url": "https://arxiv.org/abs/2311.17005", "description": "A comprehensive multimodal video-understanding benchmark converting public video annotations into multiple-choice QA spanning 20 distinct temporal task types (from perception to cognition, e.g. action sequence, counting, localization, scene/character order), specifically targeting reasoning that requires multiple frames rather than a single static image; a high score indicates strong temporal/video reasoning across diverse task categories.", "labs": []}, {"name": "LVBench", "models": 15, "categories": ["long_context"], "primary": "long_context", "secondary": null, "last_seen": "2026-08-13", "hf_dataset": "zai-org/LVBench", "implementation_url": "https://huggingface.co/datasets/zai-org/LVBench", "paper_url": "https://arxiv.org/abs/2406.08035", "description": "A long-video-understanding benchmark built from hours-long videos (TV series, sports broadcasts, surveillance footage, etc.) with multiple-choice questions that require multi-hop temporal reasoning and information retrieval across distant, disparate segments of the same video; a high score means a model can track and reason over extremely long temporal context in video.", "labs": []}, {"name": "ScreenSpot Pro", "models": 8, "categories": ["perception"], "primary": "perception", "secondary": null, "last_seen": "2026-08-09", "hf_dataset": "likaixin/ScreenSpot-Pro", "implementation_url": "https://huggingface.co/datasets/likaixin/ScreenSpot-Pro", "paper_url": "https://arxiv.org/abs/2504.07981", "description": "A GUI-grounding benchmark testing whether a model can localize the correct UI element on a screenshot given a natural-language instruction, using 1,581 authentic high-resolution screenshots from 23 professional desktop applications across 5 industries and 3 operating systems; a high score means accurate fine-grained visual grounding even for small targets in dense, high-resolution professional UIs β€” early models score below 20% accuracy.", "labs": ["meta"]}, {"name": "SLAKE", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-02-28", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "PMC-VQA", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-02-28", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "MedXpertQA-MM", "models": 12, "categories": ["reasoning"], "primary": "reasoning", "secondary": null, "last_seen": "2026-05-23", "hf_dataset": "TsinghuaC3I/MedXpertQA", "implementation_url": "https://huggingface.co/datasets/TsinghuaC3I/MedXpertQA", "paper_url": "https://arxiv.org/abs/2501.18362", "description": "The multimodal track of MedXpertQA, an expert-level medical benchmark; MM presents ~2,010 exam-style questions (5 answer choices) that pair clinical images with rich textual context like patient records and exam findings across 17 specialties and 11 body systems, requiring joint image understanding and complex clinical reasoning rather than simple image-caption QA; a high score indicates strong expert-level multimodal medical reasoning, distinct from the text-only track of the same benchmark.", "labs": []}, {"name": "HLE Verified", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-13", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "DeepPlanning", "models": 6, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-15", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "Tool-Decathlon", "models": 9, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": null, "implementation_url": "https://github.com/hkust-nlp/Toolathlon", "paper_url": "https://arxiv.org/abs/2510.25726", "description": "Also known as Toolathlon: an agentic tool-use benchmark spanning 32 realistic software applications and 600+ tools (Google Calendar, Notion, WooCommerce, Kubernetes, BigQuery, etc.) with long-horizon, execution-based evaluation of language agents; a high success rate means an agent can reliably plan and execute diverse multi-step, real-world tool-calling tasks β€” current state-of-the-art models achieve under 40% success.", "labs": []}, {"name": "MCP-Mark", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-15", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "HLE w/ Tools", "models": 27, "categories": ["knowledge", "agentic"], "primary": "knowledge", "secondary": "agentic", "last_seen": "2026-08-18", "hf_dataset": "cais/hle", "implementation_url": "https://huggingface.co/datasets/cais/hle", "paper_url": "https://arxiv.org/abs/2501.14249", "description": "This is not a separate dataset but the tool-augmented evaluation condition of Humanity's Last Exam (HLE), a 2,500-question expert-level closed-ended benchmark spanning dozens of academic subjects (multiple-choice and short-answer, ~14% multimodal); in the 'w/ Tools' setting the model is given access to web browsing/code execution rather than being closed-book, so a high score reflects both frontier academic knowledge/reasoning and effective tool use.", "labs": ["anthropic", "deepseek", "kimi", "qwen", "zai"]}, {"name": "OJBench", "models": 9, "categories": ["coding"], "primary": "coding", "secondary": null, "last_seen": "2026-04-14", "hf_dataset": "He-Ren/OJBench_testdata", "implementation_url": "https://huggingface.co/datasets/He-Ren/OJBench_testdata", "paper_url": "https://arxiv.org/abs/2506.16395", "description": "A competition-level code-generation benchmark of 232 real competitive-programming problems drawn from China's National Olympiad in Informatics and the ICPC, requiring Python or C++ solutions graded by actually executing them against the problems' full test suites; a high score means a model can produce correct, efficient solutions to hard olympiad/ICPC-level algorithmic problems, which current frontier reasoning models still largely fail.", "labs": []}, {"name": "FullStackBench", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-02-24", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "We-Math", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-02-27", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "LingoQA", "models": 5, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-02-27", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "AndroidWorld", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-05T08:22:59+00:00", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "OSWorld-Verified", "models": 16, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-08-09", "hf_dataset": "hud-evals/OSWorld-Verified", "implementation_url": "https://github.com/xlang-ai/OSWorld", "paper_url": "https://arxiv.org/abs/2404.07972", "description": "A human-verified, bug-fixed subset of OSWorld in which multimodal agents complete free-form, multi-application tasks (office suites, browsers, IDEs, file/system operations) inside real Ubuntu desktop VM environments, with success checked by automated rule-based evaluators; a high score means an agent can reliably plan and execute long-horizon, real-world GUI workflows the way a human user would. OSWorld-Verified is an updated task/infrastructure release announced via a blog post rather than a separate paper β€” the paper link is for the original OSWorld benchmark it verifies.", "labs": ["anthropic", "kimi", "meta"]}, {"name": "BabyVision", "models": 8, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["kimi"]}, {"name": "V*", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-05-23", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "TIR-Bench", "models": 5, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-02-27", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "SkillsBench", "models": 6, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["meta", "qwen"]}, {"name": "Claw-Eval", "models": 12, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-08-18", "hf_dataset": "claw-eval/Claw-Eval", "implementation_url": "https://huggingface.co/datasets/claw-eval/Claw-Eval", "paper_url": "https://arxiv.org/abs/2604.06132", "description": "An end-to-end, trajectory-aware benchmark for evaluating LLM-based autonomous agents acting in realistic environments: 300 human-verified tasks across general service orchestration, multimodal perception/creation, and multi-turn professional dialogue, graded via execution traces, audit logs, and environment snapshots against fine-grained rubric items; a high score means the agent reliably and safely completes real-world, tool-using workflows end-to-end.", "labs": []}, {"name": "TAU3-Bench", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-05", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "MCP-Atlas", "models": 19, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-08-18", "hf_dataset": "ScaleAI/MCP-Atlas", "implementation_url": "https://huggingface.co/datasets/ScaleAI/MCP-Atlas", "paper_url": "https://arxiv.org/abs/2602.00933", "description": "MCP-Atlas is a large-scale agentic tool-use benchmark of 1,000 human-authored natural-language tasks spanning 36 real (non-mocked) Model Context Protocol servers and 220 tools, requiring agents to discover relevant tools among distractors and orchestrate multi-step, cross-server workflows; responses are scored against atomic, claim-level rubrics grounded in tool outputs, so a high score reflects robust real-world tool discovery, orchestration, and error recovery rather than mere function-calling syntax correctness.", "labs": ["anthropic", "kimi", "meta"]}, {"name": "HumanEval+", "models": 18, "categories": ["coding"], "primary": "coding", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "evalplus/humanevalplus", "implementation_url": "https://huggingface.co/datasets/evalplus/humanevalplus", "paper_url": "https://arxiv.org/abs/2305.01210", "description": "An augmented, more rigorous version of OpenAI's HumanEval code-generation benchmark, where the EvalPlus project expanded per-problem test cases roughly 80x using LLM- and mutation-based test generation; a high pass@1 score means generated Python function implementations hold up against far more edge cases than the original 164-problem HumanEval suite tested, exposing previously undetected incorrect solutions.", "labs": []}, {"name": "MBPP", "models": 24, "categories": ["coding"], "primary": "coding", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "google-research-datasets/mbpp", "implementation_url": "https://huggingface.co/datasets/google-research-datasets/mbpp", "paper_url": "https://arxiv.org/abs/2108.07732", "description": "MBPP (Mostly Basic Python Problems, Austin et al., 2021) is a set of about 1,000 crowd-sourced, entry-level Python programming problems, each with a natural-language task description, a reference solution, and automated test cases; a high score means a model can reliably generate correct, executable Python code that passes the held-out tests for basic programming tasks.", "labs": []}, {"name": "MBPP+", "models": 19, "categories": ["coding"], "primary": "coding", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "evalplus/mbppplus", "implementation_url": "https://huggingface.co/datasets/evalplus/mbppplus", "paper_url": "https://arxiv.org/abs/2305.01210", "description": "An augmented, more rigorous version of the MBPP (Mostly Basic Python Problems) code-generation benchmark, where the EvalPlus project used LLM- and mutation-based test generation to expand the test cases per problem roughly 35x; a high pass@1 score means generated Python solutions are actually functionally correct under much stricter testing, not just passing the original benchmark's sparse/weak tests.", "labs": []}, {"name": "RACE", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-27", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "Global-MMLU-Lite", "models": 15, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-07-14", "hf_dataset": "CohereLabs/Global-MMLU-Lite", "implementation_url": "https://huggingface.co/datasets/CohereLabs/Global-MMLU-Lite", "paper_url": "https://arxiv.org/abs/2412.03304", "description": "A balanced, human-translated 'lite' subset of Global-MMLU covering 15-18 languages, evenly split between culturally-sensitive and culturally-agnostic multiple-choice knowledge/reasoning questions adapted from MMLU; it enables fast multilingual evaluation, and a high score indicates knowledge and reasoning ability that holds up consistently across languages and cultural contexts rather than being an artifact of machine-translation quality.", "labs": []}, {"name": "DROP", "models": 15, "categories": ["reasoning"], "primary": "reasoning", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": "ucinlp/drop", "implementation_url": "https://huggingface.co/datasets/ucinlp/drop", "paper_url": "https://arxiv.org/abs/1903.00161", "description": "A reading-comprehension benchmark of roughly 96K crowdsourced, adversarially-created questions over Wikipedia passages that require discrete reasoning operations, such as addition, counting, sorting, and comparison, over numbers, dates, and entities mentioned in the text, rather than simple span extraction; a high F1/exact-match score indicates a model can combine multiple pieces of evidence and perform correct numerical/symbolic operations to answer.", "labs": []}, {"name": "CMMLU", "models": 10, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-05-21", "hf_dataset": "haonan-li/cmmlu", "implementation_url": "https://huggingface.co/datasets/haonan-li/cmmlu", "paper_url": "https://arxiv.org/abs/2306.09212", "description": "Chinese Massive Multitask Language Understanding: a multiple-choice, MMLU-style benchmark covering 67 subjects from elementary to professional level, including China-specific domains such as Chinese history, law, and the civil-service exam; a high score indicates broad, accurate knowledge and reasoning ability in the Chinese language and cultural context.", "labs": []}, {"name": "Global-MMLU", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-27", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "HMMT", "models": 5, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-22", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "GraphWalks", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-05-23", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "ARC-Easy", "models": 6, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "Minerva Math", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-05", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "PIQA", "models": 21, "categories": ["commonsense"], "primary": "commonsense", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": "ybisk/piqa", "implementation_url": "https://huggingface.co/datasets/ybisk/piqa", "paper_url": "https://arxiv.org/abs/1911.11641", "description": "Physical Interaction QA (PIQA) is a binary-choice commonsense reasoning benchmark where, given a goal, a model must pick which of two candidate solutions (inspired by everyday/instructables.com-style scenarios) correctly achieves it through physical manipulation of everyday objects; a high score indicates good physical-world commonsense reasoning.", "labs": []}, {"name": "CommonsenseQA", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-05-21", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "SocialIQA", "models": 6, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-05-21", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "DeepMind Math", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-06", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "AlpacaEval 2.0", "models": 5, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-06", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "Terminal-Bench 2.1", "models": 33, "categories": ["coding", "agentic"], "primary": "coding", "secondary": "agentic", "last_seen": "2026-08-18", "hf_dataset": "junlin-modal/terminal-bench-2.1", "implementation_url": "https://github.com/harbor-framework/terminal-bench-2-1", "paper_url": "https://www.tbench.ai/news/terminal-bench-2-1", "description": "Terminal-Bench 2.1 evaluates AI agents on 89 realistic, complex command-line tasks (software engineering, ML, security, data processing, scientific computing, sysadmin) executed inside sandboxed container environments; it is a reliability-focused revision of Terminal-Bench 2.0 (roughly two dozen tasks fixed for broken external dependencies, misconfigured resource budgets, or instructions/tests that didn't match), and a high score means an agent can autonomously drive a real terminal to complete multi-step technical work. No dedicated Hugging Face dataset exists for the 2.1 task set; it is distributed via GitHub.", "labs": ["anthropic", "deepseek", "kimi", "meta", "openai", "qwen", "zai"]}, {"name": "Terminal-Bench Lite", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-21", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "CursorBench", "models": 5, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-08", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["anthropic"]}, {"name": "FrontierCode", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-30", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["anthropic"]}, {"name": "Online-Mind2Web", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-30", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["anthropic"]}, {"name": "Legal Agent Benchmark", "models": 5, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-08", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["anthropic"]}, {"name": "Finance Agent v2", "models": 6, "categories": ["agentic", "domain_specific"], "primary": "agentic", "secondary": "domain_specific", "last_seen": "2026-07-09", "hf_dataset": "vals-ai/finance_agent_benchmark", "implementation_url": "https://www.vals.ai/benchmarks/fabv2", "paper_url": "https://arxiv.org/abs/2508.00828", "description": "Finance Agent v2 (Vals AI) tests whether an agent can do entry-level financial-analyst work: 927 expert-reviewed questions requiring multi-step research over real SEC 10-K/10-Q filings, split into public/private-validation/held-out sets. It builds on Finance Agent v1.1 and uses a naive chunk-retrieval harness rather than a bespoke pipeline, so results reflect general agentic financial-research capability. A high score means accurate automation of expensive, repetitive analyst-grade research.", "labs": ["anthropic", "kimi"]}, {"name": "CoCounsel Legal", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-30", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["anthropic"]}, {"name": "USAMO", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-30", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["anthropic"]}, {"name": "GDPval (vision, no tools)", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-30", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["anthropic"]}, {"name": "Automated Behavioral Audit", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-30", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["anthropic"]}, {"name": "BioMysteryBench", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-30", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["anthropic"]}, {"name": "ExploitBench", "models": 6, "categories": ["coding", "safety"], "primary": "coding", "secondary": "safety", "last_seen": "2026-08-14", "hf_dataset": null, "implementation_url": "https://exploitbench.ai/", "paper_url": "https://arxiv.org/abs/2605.14153", "description": "ExploitBench reframes vulnerability exploitation as a 16-flag capability ladder β€” from triggering a crash through sandbox primitives, arbitrary read/write, control-flow hijack, to arbitrary code execution β€” instead of a binary success/fail. It's instantiated on 41 real, hardened bugs in V8 (Chrome's JS engine), with each flag checked by a deterministic oracle. Public models routinely crash the target but rarely achieve full code execution, so a high score measures how far up the real exploitation chain a model's agent climbs, not just bug discovery.", "labs": ["anthropic", "openai", "zai"]}, {"name": "Firefox Exploit Development", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-30", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["anthropic"]}, {"name": "OSS-Fuzz", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-30", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["anthropic"]}, {"name": "CyScenarioBench", "models": 5, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-09", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["anthropic"]}, {"name": "CyberGym", "models": 15, "categories": ["agentic", "safety"], "primary": "agentic", "secondary": "safety", "last_seen": "2026-08-14", "hf_dataset": "sunblaze-ucb/cybergym", "implementation_url": "https://huggingface.co/datasets/sunblaze-ucb/cybergym", "paper_url": "https://arxiv.org/abs/2506.02548", "description": "A large-scale offensive-security benchmark of 1,507 real-world vulnerability-reproduction tasks drawn from 188 open-source C/C++ projects (sourced via OSS-Fuzz), where an AI agent is given a vulnerability description and codebase and must produce a proof-of-concept input that triggers the bug; a high score indicates strong autonomous vulnerability-discovery/exploitation capability, which is why it's used as an AI-safety/cyber-capability evaluation (even top systems reach only ~20% success).", "labs": ["anthropic", "deepseek", "meta", "zai"]}, {"name": "Over-refusal rate", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-30", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["anthropic"]}, {"name": "SWE Atlas", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-23", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "SimpleQA", "models": 25, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": "basicv8vc/SimpleQA", "implementation_url": "https://huggingface.co/datasets/basicv8vc/SimpleQA", "paper_url": "https://arxiv.org/abs/2411.04368", "description": "SimpleQA is OpenAI's short-form factuality benchmark of ~4,326 diverse, fact-seeking questions each with a single, verifiable, unambiguous answer; it measures a model's factual accuracy and calibration (whether it knows what it doesn't know) on easy-to-grade but often hallucination-prone questions.", "labs": []}, {"name": "C-SimpleQA", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "IOI 2025", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-03", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "AGIEval", "models": 14, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-06-03", "hf_dataset": "baber/agieval", "implementation_url": "https://huggingface.co/datasets/baber/agieval", "paper_url": "https://arxiv.org/abs/2304.06364", "description": "A human-centric benchmark built from 20 real official standardized exams for humans (e.g., SAT, LSAT, Chinese Gaokao, law school admission and lawyer qualification tests, civil service exams, math competitions), used to measure a model's general cognitive and problem-solving ability on the same tasks and against the same difficulty humans face; a high score means performance comparable to or exceeding human test-takers on these exams.", "labs": []}, {"name": "SimpleQA Verified", "models": 6, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-14", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "Apex", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-28", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "LOFT", "models": 9, "categories": ["long_context"], "primary": "long_context", "secondary": null, "last_seen": "2026-05-23", "hf_dataset": null, "implementation_url": "https://github.com/google-deepmind/loft", "paper_url": "https://arxiv.org/abs/2406.13121", "description": "Google DeepMind's Long-Context Frontiers benchmark, repurposing 35 existing datasets (retrieval, RAG, multi-hop QA, many-shot in-context learning, SQL-style database querying, across text, visual, and audio modalities) into a long-context format with contexts up to 1M+ tokens; a high score indicates a long-context model can match or subsume specialized retrieval/RAG/SQL pipelines using only in-context reasoning over the raw corpus.", "labs": []}, {"name": "MRCR v2", "models": 13, "categories": ["long_context"], "primary": "long_context", "secondary": null, "last_seen": "2026-08-13", "hf_dataset": null, "implementation_url": "https://github.com/google-deepmind/eval_hub/tree/master/eval_hub/mrcr_v2", "paper_url": "https://arxiv.org/abs/2409.12640", "description": "Google DeepMind's open-sourced 'multi-round coreference resolution' long-context benchmark: a synthetic multi-turn chat is built from many distinct assistant responses to near-identical requests, and the model must locate and exactly reproduce one specific earlier response among similar distractors, at context lengths scaling up to 8M tokens; a high score (edit-distance match) means accurate needle retrieval despite many confusable distractors. Distinct from OpenAI's separately released 'openai/mrcr' dataset built on the same task design.", "labs": ["qwen"]}, {"name": "FLEURS", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-05-23", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "CoVoST", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-05-23", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "InfographicVQA", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-05-23", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "BLINK", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-14", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "BigBench Extra Hard", "models": 5, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-05-23", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "MTOB", "models": 6, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-05-23", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "LMArena", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "GDPval", "models": 5, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-03", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "ARC-AGI-2", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "HealthBench", "models": 6, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-08T01:50:52+00:00", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["qwen"]}, {"name": "HealthBench Hard", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-23", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "HealthBench Consensus", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-23", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "Cyber Range", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-25", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["openai"]}, {"name": "Agent's Last Exam", "models": 7, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-08-14", "hf_dataset": null, "implementation_url": "https://agents-last-exam.org", "paper_url": "https://arxiv.org/abs/2606.05405", "description": "Agents' Last Exam (ALE), from UC Berkeley RDI with 250+ industry experts: a living benchmark of long-horizon, economically valuable, real-world knowledge-work tasks with verifiable outcomes, organized into 55 sub-industries across 13 clusters (O*NET/SOC 2018 taxonomy) and growing toward 5,000 tasks; a high score means an agent can complete genuine professional deliverables end to end, not just narrow coding puzzles β€” even frontier models clear only a low single-digit percentage of the hardest tier.", "labs": ["deepseek", "kimi", "openai", "qwen", "zai"]}, {"name": "ExploitGym", "models": 3, "categories": ["coding", "safety"], "primary": "coding", "secondary": "safety", "last_seen": "2026-08-14", "hf_dataset": null, "implementation_url": "https://github.com/sunblaze-ucb/exploitgym", "paper_url": "https://arxiv.org/abs/2605.11086", "description": "ExploitGym is a follow-on to CyberGym focused specifically on exploit development rather than vulnerability reproduction, spanning 869 real, now-patched vulnerabilities in OSS-Fuzz, V8, and the Linux kernel. Agents are scored on tasks completed within fixed time budgets (2h/6h), normalized by each model's own throughput. A high score means an agent can reliably turn a known vulnerability into a working exploit at scale.", "labs": ["openai", "zai"]}, {"name": "CRUXEval-O", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-06", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "SALAD-Bench", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-06", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "AttaQ", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-06", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "PinchBench", "models": 5, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-11", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "OpenBookQA", "models": 6, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-03", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "PaperBench", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-08T01:50:52+00:00", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["qwen"]}, {"name": "AdvancedIF", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-04-14", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "DeepSearchQA", "models": 7, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["kimi", "meta"]}, {"name": "AA-Omniscience", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-11", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "CritPt", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-07", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["kimi"]}, {"name": "MathArena Apex", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-02", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "USAMO 2026", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-02", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "StrongReject", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-07-14", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "DeepSWE", "models": 8, "categories": ["coding", "agentic"], "primary": "coding", "secondary": "agentic", "last_seen": "2026-08-18", "hf_dataset": "datacurve/deep-swe", "implementation_url": "https://github.com/datacurve-ai/deep-swe", "paper_url": "https://arxiv.org/abs/2607.07946", "description": "DeepSWE measures frontier coding agents on 113 original, long-horizon software engineering tasks written from scratch (not adapted from existing PRs or commits) spanning 91 repositories across TypeScript, Go, Python, JavaScript, and Rust. This is the original v1 task set; v1.1 (tracked separately) keeps the same tasks but grades submissions in a clean, isolated environment for more reproducible scoring.", "labs": ["deepseek", "kimi"]}, {"name": "Apex-Agent", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["kimi"]}, {"name": "WildClawBench", "models": 4, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-09T07:33:17+00:00", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["meta"]}, {"name": "C3", "models": 8, "categories": ["knowledge"], "primary": "knowledge", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": "dataset-org/c3", "implementation_url": "https://huggingface.co/datasets/dataset-org/c3", "paper_url": "https://arxiv.org/abs/1904.09679", "description": "C^3 ('Investigating Prior Knowledge for Challenging Chinese Machine Reading Comprehension'): the first free-form multiple-choice Chinese machine reading comprehension dataset, with documents (dialogues or mixed-genre texts) drawn from Chinese-as-a-second-language exams paired with multiple-choice questions; a high score means correctly selecting the exam-style answer, and most questions require knowledge beyond the document itself. The short name is ambiguous β€” this entry is specifically the Sun et al. 2019 CSL reading-comprehension dataset.", "labs": []}, {"name": "LiveCodeBench-Base", "models": 8, "categories": ["coding"], "primary": "coding", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": "livecodebench/code_generation_lite", "implementation_url": "https://huggingface.co/datasets/livecodebench/code_generation_lite", "paper_url": "https://arxiv.org/abs/2403.07974", "description": "A base-model evaluation configuration of LiveCodeBench, using the same contamination-resistant competitive-programming problems but prompting pretrained/non-instruction-tuned models with a few-shot code-completion-style template instead of chat instructions; a high score means the base model can generate functionally correct code on recent problems unlikely to be in its pretraining data.", "labs": []}, {"name": "SWE-rebench", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "Vending-Bench 2", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "CC-Bench-V2", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "ZMultiTransBench", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "ToolCall-Badcase", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-06-16", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}, {"name": "OSWorld 2.0", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-13", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["kimi"]}, {"name": "JobBench", "models": 4, "categories": ["agentic", "domain_specific"], "primary": "agentic", "secondary": "domain_specific", "last_seen": "2026-08-08T01:50:52+00:00", "hf_dataset": null, "implementation_url": "https://arxiv.org/abs/2605.26329", "paper_url": "https://arxiv.org/abs/2605.26329", "description": "JobBench evaluates whether agents can do the specific work professionals actually want delegated: 130 tasks across 35 occupations, each a cluttered workspace of heterogeneous real reference files, elicited via a preference survey of what people want automated. It's graded by chained rubrics with thousands of binary criteria that only award credit when every step holds, penalizing plausible-but-broken multi-step work. The best setup (Claude Opus 4.7 under Claude Code) reaches only 45.9%, so a high score reflects reliable, delegation-grade professional work.", "labs": ["kimi", "qwen"]}, {"name": "Toolathlon-Verified", "models": 10, "categories": ["agentic"], "primary": "agentic", "secondary": null, "last_seen": "2026-08-18", "hf_dataset": "hkust-nlp/Toolathlon-Verified_Trajectories", "implementation_url": "https://github.com/hkust-nlp/Toolathlon", "paper_url": "https://arxiv.org/abs/2510.25726", "description": "Toolathlon (published as \"The Tool Decathlon\") tests language agents across 32 real software applications and 604 tools β€” from Google Calendar and Notion to Kubernetes and BigQuery β€” with 108 manually crafted tasks requiring roughly 20 turns of multi-app tool use in realistic, stateful environments. Each task is graded by a dedicated verification script against the resulting environment state; \"Verified\" denotes results obtained via the official evaluation service. Even the best model (Claude Sonnet 4.5) reaches only a 38.6% success rate, so a high score reflects genuine multi-app, long-horizon tool orchestration.", "labs": ["deepseek", "kimi", "qwen", "zai"]}, {"name": "SWE Marathon", "models": 3, "categories": ["coding"], "primary": "coding", "secondary": null, "last_seen": "2026-08-14", "hf_dataset": null, "implementation_url": "https://github.com/abundant-ai/swe-marathon", "paper_url": "https://arxiv.org/abs/2606.07682", "description": "SWE-Marathon is a 20-task benchmark of ultra-long-horizon software engineering work β€” building compilers, optimizing kernels, standing up production services β€” where logged agent attempts average 27.2M tokens, far beyond single-PR benchmarks like SWE-bench. Each task ships a unique executable environment, a human-written reference solution, and a multi-layer verification suite; v1.1 tightened verification and closed internet access during execution. Current frontier agents solve fewer than 30% of tasks.", "labs": ["kimi", "zai"]}, {"name": "FrontierSWE", "models": 3, "categories": ["coding"], "primary": "coding", "secondary": null, "last_seen": "2026-08-14", "hf_dataset": null, "implementation_url": "https://github.com/Proximal-Labs/frontier-swe", "paper_url": "https://www.proximal.so/blog/frontierswe", "description": "FrontierSWE (Proximal) tests coding agents on ultra-long-horizon technical challenges spanning performance engineering, computational science, and ML research β€” deliberately harder than PR-sized SWE-bench tasks. It's scored via a \"dominance\" metric (win rate versus a random opponent per task) plus average rank, at mean@5 and best@5. A high score reflects an agent's ability to both correctly solve and meaningfully optimize genuinely difficult engineering problems. No arXiv paper exists; documented only on Proximal's own site.", "labs": ["kimi", "qwen", "zai"]}, {"name": "GDPval-AA v2", "models": 7, "categories": ["agentic", "domain_specific"], "primary": "agentic", "secondary": "domain_specific", "last_seen": "2026-08-14", "hf_dataset": "openai/gdpval", "implementation_url": "https://artificialanalysis.ai/evaluations/gdpval-aa", "paper_url": "https://artificialanalysis.ai/evaluations/gdpval-aa", "description": "GDPval-AA v2 is Artificial Analysis's evaluation harness around OpenAI's GDPval task set β€” 220 real, economically valuable tasks across 44 occupations and 9 industries, judged via blind pairwise comparison (Elo re-baselined so 1000 represents human-professional performance). Models get shell/web access via an agentic loop and must produce real work products (documents, slides, spreadsheets, diagrams); v2 raised the turn limit from 100 to 250 and added a rotating panel of frontier-model judges. A score above 1000 Elo means a model's output beat a human professional's more often than not.", "labs": ["kimi", "meta", "zai"]}, {"name": "AutomationBench", "models": 6, "categories": ["agentic", "domain_specific"], "primary": "agentic", "secondary": "domain_specific", "last_seen": "2026-08-14", "hf_dataset": null, "implementation_url": "https://github.com/zapier/AutomationBench", "paper_url": "https://arxiv.org/abs/2604.18934", "description": "AutomationBench (Zapier) evaluates agents on 657 realistic cross-application business workflows across sales, marketing, operations, support, finance, and HR, built from patterns seen across Zapier's own automation traffic. Agents must discover and call the right REST endpoints among 47 tools across 40 simulated apps (Gmail, Salesforce, Slack, Jira, HubSpot, etc.), navigating irrelevant records and layered business rules. Grading is fully programmatic against final environment state (no LLM-as-judge), so a high score means an agent can autonomously complete real multi-step business operations.", "labs": ["deepseek", "kimi", "qwen", "zai"]}, {"name": "TAU3-Bench Banking", "models": 5, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-11", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": ["kimi", "meta"]}, {"name": "DeepSWE 1.1", "models": 4, "categories": ["coding", "agentic"], "primary": "coding", "secondary": "agentic", "last_seen": "2026-08-14", "hf_dataset": "datacurve/deep-swe", "implementation_url": "https://github.com/datacurve-ai/deep-swe", "paper_url": "https://arxiv.org/abs/2607.07946", "description": "DeepSWE measures frontier coding agents on 113 original, long-horizon software engineering tasks written from scratch (not adapted from existing PRs or commits) spanning 91 repositories across TypeScript, Go, Python, JavaScript, and Rust; v1.1 keeps the same task set as v1 but grades each agent's committed code in a clean, isolated environment for more reproducible and auditable scoring.", "labs": ["qwen", "zai"]}, {"name": "SWE Atlas - QnA", "models": 3, "categories": ["uncategorized"], "primary": "uncategorized", "secondary": null, "last_seen": "2026-08-18", "hf_dataset": null, "implementation_url": null, "paper_url": null, "description": null, "labs": []}], "n_hidden": 348, "n_total": 575, "category_colors": {"knowledge": {"hex": "#2a78d6", "hue": 213}, "coding": {"hex": "#1baf7a", "hue": 159}, "agentic": {"hex": "#eda100", "hue": 41}, "reasoning": {"hex": "#0a9b9b", "hue": 180}, "math": {"hex": "#4f7d00", "hue": 82}, "perception": {"hex": "#e34948", "hue": 0}, "long_context": {"hex": "#eb6834", "hue": 17}, "instruction_following": {"hex": "#c2489d", "hue": 318}, "commonsense": {"hex": "#8a6d3b", "hue": 38}, "domain_specific": {"hex": "#5b6ee1", "hue": 231}, "alignment": {"hex": "#4a3aa7", "hue": 249}, "safety": {"hex": "#a33a52", "hue": 346}, "translation": {"hex": "#e87ba4", "hue": 337}, "writing": {"hex": "#8a7000", "hue": 49}, "uncategorized": {"hex": "#86847d", "hue": 47}}, "category_order": ["agentic", "knowledge", "coding", "reasoning", "math", "perception", "long_context", "instruction_following", "commonsense", "domain_specific", "alignment", "safety", "translation", "writing", "uncategorized"]}</script>
<script>
const DATA = JSON.parse(document.getElementById('data-blob').textContent);
const CARDS = DATA.cards;
const CATS = DATA.category_colors;
const CATEGORY_ORDER = DATA.category_order;
const LAB_ICONS = JSON.parse(document.getElementById('lab-icons').textContent);
const OFFICIAL_LEADERBOARDS = new Set(["cais/hle", "openai/gsm8k", "llamaindex/ExtractBench", "IntelligenceLab/Long-Horizon-Terminal-Bench", "Idavidrein/gpqa", "ScaleAI/SWE-bench_Pro", "datacurve/deep-swe", "SWE-bench/SWE-bench_Verified", "hf-audio/open-asr-leaderboard", "allenai/olmOCR-bench", "MathArena/aime_2026", "tiiuae/PBench", "internlm/WildClawBench", "MMMU/MMMU_Pro", "LEXam-Benchmark/LEXam", "VLABench/vlabench_primitive_ft_lerobot_video", "mercor/apex-agents", "Delores-Lin/MDPBench", "llamaindex/ParseBench", "benchflow/skillsbench", "mteb/arguana", "TIGER-Lab/MMLU-Pro", "likaixin/ScreenSpot-Pro", "nvidia/compute-eval", "SWE-bench/SWE-bench_Multilingual", "mercor/ACE", "mercor/APEX-v1-extended", "PaddlePaddle/Real5-OmniDocBench", "FutureMa/EvasionBench", "harborframework/terminal-bench-2.0", "MathArena/hmmt_feb_2026", "mteb/BRIGHT", "collinear-ai/yc-bench", "claw-eval/Claw-Eval", "InternScience/ResearchClawBench", "MME-Benchmarks/Video-MME-v2", "ChrisHayduk/nanofold-public", "actava/chi-bench", "meituan-longcat/WBench", "ARTPARK-IISc/Vaani-Benchmark-V1.0", "crosbylegal/RedlineBench", "LiquidAI/ifstruct-v1.0"]); // datasets tagged benchmark:official on the Hub
const LAB_ORDER = ['openai', 'anthropic', 'kimi', 'zai', 'deepseek', 'qwen', 'meta']; // fixed display order
// rarity tier = how many major labs' latest release use this benchmark (0-6).
// Name, max tilt (deg), and hover scale all ramp together, PokΓ©mon-TCG style.
const RARITY = [
 { name: null, tilt: 10, scale: 1.045 }, // 0 Β· Common
 { name: 'Uncommon', tilt: 11, scale: 1.045 }, // 1
 { name: 'Rare', tilt: 12, scale: 1.05 }, // 2
 { name: 'Rare Holo', tilt: 14, scale: 1.05 }, // 3
 { name: 'Ultra Rare', tilt: 16, scale: 1.06 }, // 4
 { name: 'Rainbow Rare', tilt: 18, scale: 1.07 }, // 5
 { name: 'Secret Rare ✦', tilt: 20, scale: 1.09 }, // 6 · every major lab
];
document.getElementById('hidden-note').textContent =
 `${DATA.n_hidden} of ${DATA.n_total} tracked benchmarks fall outside that window and are left out of the deck.`;
const grid = document.getElementById('grid');
const catPillsEl = document.getElementById('cat-pills');
const searchEl = document.getElementById('search');
const sortEl = document.getElementById('sortsel');
const countNote = document.getElementById('count-note');
let state = { category: 'All', search: '', sort: 'models' };
function esc(s) {
 return String(s).replace(/[&<>"']/g, c => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
}
// ---- build category filter pills, colored by category accent ----
const presentCats = [...new Set(CARDS.map(c => c.primary))]
 .sort((a, b) => CATEGORY_ORDER.indexOf(a) - CATEGORY_ORDER.indexOf(b));
['All', ...presentCats].forEach(cat => {
 const btn = document.createElement('button');
 btn.className = 'pill' + (cat === 'All' ? ' on' : '');
 btn.textContent = cat === 'All' ? 'All' : cat.replace(/_/g, ' ');
 btn.dataset.cat = cat;
 if (cat !== 'All') btn.style.setProperty('--pill-c', CATS[cat].hex);
 btn.addEventListener('click', () => {
 state.category = cat;
 [...catPillsEl.children].forEach(b => b.classList.toggle('on', b === btn));
 render();
 });
 catPillsEl.appendChild(btn);
});
searchEl.addEventListener('input', () => { state.search = searchEl.value.toLowerCase(); render(); });
sortEl.addEventListener('change', () => { state.sort = sortEl.value; render(); });
// ---- build every card element once; filtering/sorting only toggles class + order ----
const cardEls = CARDS.map((c, i) => {
 const cat = CATS[c.primary] || CATS.uncategorized;
 // Dual-category cards blend a second color/hue in; single-category ones
 // just repeat the primary so every gradient/color-mix below collapses to
 // one flat color with no special-casing needed.
 const cat2 = c.secondary ? (CATS[c.secondary] || cat) : cat;
 const wrap = document.createElement('div');
 wrap.className = 'card-wrap';
 const rarity = Math.min((c.labs || []).length, 6);
 const tier = RARITY[rarity];
 const card = document.createElement('div');
 card.className = 'card' + (rarity > 0 ? ` card--r${rarity}` : '');
 card.style.setProperty('--accent2', cat.hex);
 card.style.setProperty('--accent3', cat2.hex);
 card.style.setProperty('--hue', cat.hue);
 card.style.setProperty('--hue2', cat2.hue);
 card.style.setProperty('--sweep-delay', `${(i % 12) * 0.6}s`);
 const links = [];
 if (c.hf_dataset) {
 // Datasets tagged benchmark:official on the Hub render an actual ranked
 // leaderboard on that same page β€” badge it as such instead of a plain
 // dataset link when that's the case (same URL either way).
 const isLeaderboard = OFFICIAL_LEADERBOARDS.has(c.hf_dataset);
 const label = isLeaderboard ? 'πŸ† Leaderboard' : 'πŸ€— HF dataset';
 links.push(`<a href="https://huggingface.co/datasets/${esc(c.hf_dataset)}" target="_blank">${label}</a>`);
 }
 if (c.implementation_url && c.implementation_url !== (c.hf_dataset ? `https://huggingface.co/datasets/${c.hf_dataset}` : null)) {
 links.push(`<a href="${esc(c.implementation_url)}" target="_blank">Implementation</a>`);
 }
 if (c.paper_url) links.push(`<a href="${esc(c.paper_url)}" target="_blank">Paper</a>`);
 const badgeText = c.secondary
 ? `${c.primary.replace(/_/g, ' ')} + ${c.secondary.replace(/_/g, ' ')}`
 : c.primary.replace(/_/g, ' ');
 // "Used by:" row β€” one icon per major lab (of openai/anthropic/kimi/zai/
 // deepseek/qwen/meta) whose single *latest* release reports this benchmark
 // (not just any model ever from that lab), in a fixed order
 const usedBy = (c.labs || []).slice().sort((a, b) => LAB_ORDER.indexOf(a) - LAB_ORDER.indexOf(b));
 const usedByHtml = usedBy.length
 ? `<div class="card__usedby"><span class="card__usedby-label">Used by</span>
 ${usedBy.map(code => `<span class="lab-icon" title="${esc(LAB_ICONS.labels[code])}">${LAB_ICONS.icons[code]}</span>`).join('')}
 </div>`
 : '';
 const effectLayers = rarity > 0
 ? '<div class="card__foil"></div><div class="card__glare"></div>'
 : '';
 const rarityHtml = tier.name ? `<div class="card__rarity">${esc(tier.name)}</div>` : '';
 card.innerHTML = `
 ${effectLayers}
 <div class="card__body">
 <span class="card__badge">${esc(badgeText)}</span>
 ${rarityHtml}
 <div class="card__mid">
 <div class="card__title">${esc(c.name)}</div>
 <div class="card__stat"><b>${c.models}</b> model${c.models === 1 ? '' : 's'} reporting</div>
 ${c.description ? `<div class="card__desc">${esc(c.description)}</div>` : ''}
 </div>
 ${usedByHtml}
 <div class="card__foot">${links.length ? links.join('') : '<span class="none">No implementation catalogued yet</span>'}</div>
 </div>`;
 // pointer-tracked tilt for every card, intensity scaled by rarity tier;
 // foil/glare position only matters where those elements exist
 const MAX_TILT = tier.tilt;
 // --px/--py: raw pointer position (0-100), used to center the glare/vignette.
 // --bgx/--bgy: the SAME pointer position compressed into a narrow band
 // (37-63 / 33-67) before the foil layers amplify it 2.6x/3.5x β€” this is
 // the actual pokemon-cards-css mapping (their `adjust()` helper). Feeding
 // the raw 0-100 pointer into that amplification instead (what an earlier
 // version of this page did) makes the foil swing wildly instead of
 // panning smoothly β€” it's the main reason it looked "off".
 function onMove(e) {
 const rect = card.getBoundingClientRect();
 const px = ((e.clientX - rect.left) / rect.width) * 100;
 const py = ((e.clientY - rect.top) / rect.height) * 100;
 const bgx = 37 + (px / 100) * 26;
 const bgy = 33 + (py / 100) * 34;
 const rx = (py - 50) / 50 * -MAX_TILT;
 const ry = (px - 50) / 50 * MAX_TILT;
 card.classList.add('live');
 card.style.setProperty('--px', px.toFixed(1));
 card.style.setProperty('--py', py.toFixed(1));
 card.style.setProperty('--bgx', bgx.toFixed(1));
 card.style.setProperty('--bgy', bgy.toFixed(1));
 card.style.setProperty('--op', 1);
 card.style.transform = `rotateX(${rx.toFixed(2)}deg) rotateY(${ry.toFixed(2)}deg) scale(${tier.scale})`;
 }
 function onLeave() {
 card.classList.remove('live');
 card.style.setProperty('--op', 0);
 card.style.transform = 'rotateX(0deg) rotateY(0deg) scale(1)';
 }
 card.addEventListener('pointermove', onMove);
 card.addEventListener('pointerleave', onLeave);
 card.addEventListener('pointercancel', onLeave);
 wrap.appendChild(card);
 wrap._meta = { ...c, rarity };
 wrap._card = card;
 return wrap;
});
cardEls.forEach(el => grid.appendChild(el));
function render() {
 let visible = 0;
 cardEls.forEach(el => {
 const c = el._meta;
 const catOk = state.category === 'All' || c.primary === state.category;
 const searchOk = !state.search || c.name.toLowerCase().includes(state.search);
 const show = catOk && searchOk;
 el.classList.toggle('hidden', !show);
 if (show) visible++;
 });
 const sorted = [...cardEls].sort((a, b) => {
 const A = a._meta, B = b._meta;
 if (A.rarity !== B.rarity) return B.rarity - A.rarity; // rarer cards always float to the top
 if (state.sort === 'name') return A.name.localeCompare(B.name);
 if (state.sort === 'category') {
 const r = CATEGORY_ORDER.indexOf(A.primary) - CATEGORY_ORDER.indexOf(B.primary);
 return r !== 0 ? r : A.name.localeCompare(B.name);
 }
 if (state.sort === 'labs') {
 return (B.labs || []).length - (A.labs || []).length || B.models - A.models || A.name.localeCompare(B.name);
 }
 return B.models - A.models || A.name.localeCompare(B.name); // "models"
 });
 sorted.forEach((el, i) => { el.style.order = i; });
 countNote.textContent = `${visible} benchmark${visible === 1 ? '' : 's'} shown`;
}
render();
</script>
</body>
</html>