File size: 47,112 Bytes
51c2050 0ad648e 51c2050 080683f 51c2050 0ad648e 51c2050 62020c8 51c2050 55d8114 3115c56 55d8114 51c2050 3115c56 55d8114 51c2050 080683f 8a8106a 080683f 3115c56 080683f c9a2c88 080683f 51c2050 85b14b3 51c2050 85b14b3 51c2050 85b14b3 51c2050 85b14b3 51c2050 85b14b3 51c2050 85b14b3 51c2050 85b14b3 51c2050 85b14b3 51c2050 85b14b3 51c2050 85b14b3 51c2050 85b14b3 51c2050 85b14b3 51c2050 fd26be1 080683f fd26be1 51c2050 080683f 85b14b3 51c2050 85b14b3 3115c56 51c2050 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 | /*
* Aligned AI benchmarks: single data file (the only file you need to edit).
* ---------------------------------------------------------------------------
* Loaded by index.html as a plain <script>, so it works from any static host
* (Hugging Face static Space, python -m http.server, or file://). No build step.
*
* SCHEMA
* window.BENCH_DATA = {
* meta: {
* title, badge, summary, // header text
* measured: "Sep 2026", // fixed label used in every card footer (never "today")
* preview: true, // true shows the "Preview" pill and pending notes
* headlineBoards: [ids...], // up to 4 stat boxes in the header
* strengths: [{ id, why }], // up to 3 "Where Aligned is strong" boards
* methodology: [paragraphs...]
* },
* categories: { "<Category>": { dot, tagBg, tagText, tagRing } },
* boards: [{
* id, name, category, description,
* unit: "accuracy %" | "pass@1 %",
* higherIsBetter: true,
* links: { // footer "Dataset · Paper" links (all verified HTTP 200)
* dataset: "https://huggingface.co/datasets/<org>/<name>", datasetLabel?: "Dataset (HF mirror)",
* paper: "https://arxiv.org/abs/<id>", paperLabel?: "Announcement",
* extra?: [{ label, url }] // e.g. official CSV, subset paper
* },
* fieldSize?: 20, // total models in the source ranking
* (aligned.rank?: true rank in the full source ranking when only a subset of competitors is listed)
* fieldSize?: 20, // total models in the source ranking (only the top entries are listed)
* pending: false, // true = hidden until aligned.score is filled in
* aligned: {
* score: 93.3, // number, 0-100 (null while pending)
* sample: "150 of 1,172 questions", // shown in the footer
* runId: "36425671251", // displayed as plain text provenance
* note: "" // optional caveat shown under the footer
* },
* competitors: [ // empty = "Competitor scores pending"
* { model: "GPT-5", vendor: "openai", score: 91.2,
* source: "https://... (URL of the public page the score was quoted from)",
* sourceLabel: "OpenAI system card" } // sourceLabel optional
* ]
* }]
* }
*
* vendor must be one of: openai, anthropic, google, xai, meta, mistral, deepseek, qwen, groq, other
* (matches brand/vendors/<vendor>.png). Rank "#N of M" is computed from real entries only:
* sorted by score (desc if higherIsBetter), ties share a rank, Aligned is never forced.
*/
window.BENCH_DATA = {
meta: {
title: "September 2026 Benchmarks",
badge: "Published September 28, 2026",
summary:
"Nine public benchmarks, measured on Aligned AI's production pipeline and scored by each benchmark's official grader.",
measured: "Sep 2026",
preview: false,
headlineBoards: ["graphwalks", "simpleqa", "gsm8k"],
headlineFootnote: "[2] GraphWalks: the run hit its time limit; 92 of 150 sampled questions completed and all 92 were correct. A full-run score would likely be lower. See methodology.",
headlineStats: [
{ label: "GraphWalks", value: "100%", detail: "", jump: "board-graphwalks" },
{ label: "MATH-500", value: "96.6%", detail: "", jump: "board-math-500" },
{ label: "GSM8K", value: "93.3%", detail: "", jump: "board-gsm8k" },
{ label: "Cost per answer", value: "$0.00069", detail: "", jump: "cost" }
],
strengths: [
{ id: "math-500", why: "500 high-school competition math problems, scored on the final answer. Aligned AI scores 96.6%, within 3.0 points of the top score." },
{ id: "gsm8k", why: "Grade-school math word problems that require multi-step arithmetic reasoning. Aligned AI scores 93.3%, within 2.1 points of the top score." },
{ id: "mbpp", why: "Entry-level Python programming tasks checked against test cases. Aligned AI ranks fourth, ahead of GPT-5 (high)." }
],
methodology: [
"Source of results. All Aligned AI scores were produced by the Aligned AI production pipeline available at joinaligned.ai, as measured in September 2026.",
"Evaluation method. Aligned AI was evaluated on samples drawn from each benchmark’s official test set and scored using the benchmark’s official grader or reference scoring method. Sample sizes are stated on the relevant board where recorded. Each board references the benchmark’s official dataset and, where available, its original publication.",
"Competitor data. Competitor scores are drawn from a dataset compiled by Aligned on September 28, 2026, covering leading models released in or after July 2025. Rankings reflect Aligned AI’s position relative to the models in that dataset only and are not intended as a comprehensive ranking of all available models.",
"Model coverage. Not every model has published or publicly available results for every benchmark, and some benchmarks are reported only for certain model versions, configurations or release dates. As a result, the set of competitor models shown differs from board to board, and a model\u2019s absence from a board should not be interpreted as an indication of its performance. Rankings are relative only to the models listed on each board.",
"Self-reported results. All Aligned AI scores on this page are self-reported. Each evaluation was conducted by Aligned on its own infrastructure, and no independent third party has reproduced, audited or verified these results.",
"System comparison. Aligned AI is a compound system that may combine multiple models, tools and retrieval within a single production pipeline; it is not a single standalone model. Competitor scores generally describe individual models evaluated under their publishers’ own settings. Comparisons are therefore indicative only and should not be construed as like-for-like.",
"Sampling. Aligned AI scores are based on samples of each official test set rather than the complete test set. Sample-based results are subject to statistical uncertainty and may differ from results on a full evaluation.",
"Preliminary results. Certain scores, including MBPP and TriviaQA, were obtained from preliminary evaluation runs of approximately 50 questions on an earlier version of the Aligned AI pipeline. Results from limited samples are subject to substantial variance, should be regarded as indicative estimates only, and may be materially higher or lower in subsequent or full-scale evaluations.",
"Third-party figures. Competitor scores represent a point-in-time snapshot of publicly reported results as of September 28, 2026. They were not independently re-run by Aligned and may since have been updated, corrected or withdrawn by their publishers. Evaluation conditions, including prompting, number of attempts, tool use and grader version, may vary between sources and from those used by Aligned. Aligned makes no representation as to the accuracy or completeness of third-party figures.",
"SimpleQA: Aligned AI answered with web search enabled.",
"[2] GraphWalks: the evaluation reached its time limit, and the reported score reflects the 92 of 150 sampled questions that completed.",
"No warranty. Results reflect the Aligned AI pipeline at the time of measurement and may change as the system is updated. All information on this page is provided for informational purposes only, on an “as is” basis, without representation or warranty of any kind, express or implied, including as to accuracy, completeness or future performance. Benchmark results do not guarantee performance on any particular task, use or deployment. Model names and trademarks are the property of their respective owners, and their inclusion does not imply affiliation with or endorsement by those owners."
]
},
// Tailwind category colors, matching the previous page (oklch values from its CSS).
categories: {
Knowledge: { dot: "oklch(62.3% .214 259.815)", tagBg: "oklch(93.2% .032 255.585)", tagText: "oklch(37.9% .146 265.522)", tagRing: "oklch(80.9% .105 251.813 / .7)" },
Code: { dot: "oklch(70.5% .213 47.604)", tagBg: "oklch(95.4% .038 75.164)", tagText: "oklch(40.8% .123 38.172)", tagRing: "oklch(83.7% .128 66.29 / .7)" },
Math: { dot: "oklch(69.6% .17 162.48)", tagBg: "oklch(95% .052 163.051)", tagText: "oklch(37.8% .077 168.94)", tagRing: "oklch(84.5% .143 164.978 / .7)" },
Reasoning: { dot: "oklch(76.9% .188 70.08)", tagBg: "oklch(96.2% .059 95.617)", tagText: "oklch(41.4% .112 45.904)", tagRing: "oklch(87.9% .169 91.605 / .7)" },
Science: { dot: "oklch(71.5% .143 215.221)", tagBg: "oklch(95.6% .045 203.388)", tagText: "oklch(39.8% .07 227.392)", tagRing: "oklch(86.5% .127 207.078 / .7)" },
"Long context": { dot: "oklch(65.6% .241 354.308)", tagBg: "oklch(94.8% .028 342.258)", tagText: "oklch(40.8% .153 2.432)", tagRing: "oklch(82.3% .12 346.018 / .7)" }
},
cost: {
title: "Estimated cost per general answer",
summary: "Aligned AI's running cost is lower than every model shown, including budget models. Same workload for every model: 8,717 input + 935 output tokens per answer. Lower is better.",
explainer: "One answer = a typical full written reply to a general question (avg. ~8,700 input + ~935 output tokens across 165 real answers), e.g. \u201cWhat was the first Nike running shoe?\u201d",
aligned: 0.00069,
groups: [
{ name: "Flagship & mid-tier models", rows: [
["Mistral Large 3","mistral",0.00576],["DeepSeek V4 Pro","deepseek",0.01521],["Mistral Medium 3.5","mistral",0.02009],
["Gemini 3.8 Flash","google",0.02009],["Grok 4.7","xai",0.02304],["GPT-6 Sol","openai",0.02678],["Claude Sonnet 5","anthropic",0.02678],
["Gemini 3.1 Pro Preview","google",0.02865],["Kimi K3","other",0.04018],["Claude Opus 5.5","anthropic",0.05357],
["GPT-6 Astra","openai",0.13392],["Claude Fable 5.1","anthropic",0.13392] ] },
{ name: "Budget models", rows: [
["GPT-6 Luna","openai",0.00134],["Mistral Small 4","mistral",0.00187],["DeepSeek V4.1 Flash","deepseek",0.00374],
["Gemini 3.5 Flash-Lite","google",0.00495],["Claude Haiku 4.5","anthropic",0.01339] ] }
],
footnote: "[1] Estimated, not billed. Competitor costs are calculated by applying each provider's publicly listed per-token API prices, as of September 28, 2026, to the same fixed workload (8,717 input + 935 output tokens per answer). They exclude caching, batch or volume discounts, reasoning or thinking tokens, tool and search fees, and regional pricing, and actual costs will vary with usage. Prices are a point-in-time snapshot and providers may change them at any time. Aligned AI's figure is its own estimated infrastructure cost for the same workload on its production pipeline, is self-reported, and is not a customer price. The workload is an average of 165 real Aligned AI answers to general questions; other tasks may use very different token counts."
},
boards: [
{
id: "graphwalks", name: "GraphWalks", category: "Reasoning", fieldSize: 16,
description: "Long-context multi-hop reasoning benchmark released by OpenAI in 2025. Each prompt contains a large directed graph written out as a list of edges, and the model must return either every node at an exact breadth-first-search depth from a start node or every parent of a given node. Answers are node sets graded against the ground truth.",
unit: "accuracy %", higherIsBetter: true,
links: { dataset: "https://huggingface.co/datasets/openai/graphwalks", datasetLabel: "Official dataset (OpenAI)" },
aligned: { rank: 1, score: 100, sample: "92 of 92 graded questions", runId: "36316454589",
note: "The run reached its time limit before completing the remaining 58 of 150 sampled questions." },
competitors: [{"model": "GPT-5.2 (high)", "vendor": "openai", "lab": "OpenAI", "rank": 2, "score": 98.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-5.4", "vendor": "openai", "lab": "OpenAI", "rank": 3, "score": 93.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Claude Opus 4.5", "vendor": "anthropic", "lab": "Anthropic", "rank": 4, "score": 92.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Anthropic (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-5 mini (high)", "vendor": "openai", "lab": "OpenAI", "rank": 5, "score": 85.5, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemini 3 Flash (high)", "vendor": "google", "lab": "Google", "rank": 6, "score": 84.2, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Seed2.0 Lite", "vendor": "other", "lab": "ByteDance", "rank": 7, "score": 82.5, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "ByteDance (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Claude Sonnet 4.5", "vendor": "anthropic", "lab": "Anthropic", "rank": 8, "score": 80.5, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Anthropic (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemini 3 Pro (high)", "vendor": "google", "lab": "Google", "rank": 9, "score": 79.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-5 (high)", "vendor": "openai", "lab": "OpenAI", "rank": 10, "score": 78.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-5.1 Thinking", "vendor": "openai", "lab": "OpenAI", "rank": 11, "score": 76.8, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-5.4 mini", "vendor": "openai", "lab": "OpenAI", "rank": 12, "score": 76.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-5.4 nano", "vendor": "openai", "lab": "OpenAI", "rank": 13, "score": 73.4, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Seed2.0 Pro", "vendor": "other", "lab": "ByteDance", "rank": 14, "score": 68.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "ByteDance (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Seed2.0 Mini", "vendor": "other", "lab": "ByteDance", "rank": 15, "score": 64.1, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "ByteDance (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-5 nano (high)", "vendor": "openai", "lab": "OpenAI", "rank": 16, "score": 64.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}]
},
{
id: "simpleqa", name: "SimpleQA", category: "Knowledge", fieldSize: 40,
description: "Factual-accuracy benchmark from OpenAI (Wei et al., 2024). 4,326 short, fact-seeking questions, each with a single indisputable answer verified by independent annotators. Responses are graded as correct, incorrect, or not attempted.",
unit: "accuracy %", higherIsBetter: true,
links: { dataset: "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv", datasetLabel: "Official dataset (OpenAI)", paper: "https://arxiv.org/abs/2411.04368", paperLabel: "Paper (Wei et al., 2024)" },
// TODO(Chris): add the sample size, e.g. "N of 4,326 questions". Left blank so we don't make one up.
aligned: { rank: 1, score: 81.6, sample: "", runId: "36362020662", },
competitors: [{"model": "Gemini 3.1 Pro", "vendor": "google", "lab": "Google", "rank": 2, "score": 74.8, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-6 Astra", "vendor": "openai", "lab": "OpenAI", "rank": 3, "score": 73.1, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemini 3 Pro", "vendor": "google", "lab": "Google", "rank": 4, "score": 70.5, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemini 3.8 Flash", "vendor": "google", "lab": "Google", "rank": 5, "score": 69.5, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemini 3.7 Flash", "vendor": "google", "lab": "Google", "rank": 6, "score": 69.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-5.6 Sol", "vendor": "openai", "lab": "OpenAI", "rank": 7, "score": 67.7, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemini 3.5 Flash", "vendor": "google", "lab": "Google", "rank": 8, "score": 66.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemini 3.6 Flash", "vendor": "google", "lab": "Google", "rank": 9, "score": 65.6, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Claude Opus 5", "vendor": "anthropic", "lab": "Anthropic", "rank": 10, "score": 62.2, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Anthropic (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-5", "vendor": "openai", "lab": "OpenAI", "rank": 11, "score": 55.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Grok 4.5", "vendor": "xai", "lab": "xAI", "rank": 12, "score": 51.8, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "xAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Qwen3 235B A22B Instruct 2507", "vendor": "qwen", "lab": "Alibaba", "rank": 13, "score": 50.6, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Alibaba (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Qwen3 235B A22B Thinking 2507", "vendor": "qwen", "lab": "Alibaba", "rank": 14, "score": 50.4, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Alibaba (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Grok 4", "vendor": "xai", "lab": "xAI", "rank": 15, "score": 50.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "xAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Claude Opus 4.6", "vendor": "anthropic", "lab": "Anthropic", "rank": 16, "score": 48.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Anthropic (Aligned competitor dataset, Sep 28 2026)"}]
},
{
id: "truthfulqa", name: "TruthfulQA", category: "Knowledge", fieldSize: 8,
description: "Truthfulness benchmark from the University of Oxford and OpenAI (Lin, Hilton and Evans, 2021). 817 questions across 38 categories, including health, law, finance and politics, written so that a model repeating common misconceptions will answer falsely.",
unit: "accuracy %", higherIsBetter: true,
links: { dataset: "https://huggingface.co/datasets/truthfulqa/truthful_qa", paper: "https://arxiv.org/abs/2109.07958", paperLabel: "Paper (Lin et al., 2021)" },
aligned: { rank: 3, score: 75.0, sample: "100 questions", runId: "36372933184" },
competitors: [{"model": "MAI-Thinking-1", "vendor": "other", "lab": "Microsoft", "rank": 1, "score": 88.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Microsoft (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Claude Sonnet 4.6", "vendor": "anthropic", "lab": "Anthropic", "rank": 1, "score": 88.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Anthropic (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemma 4 26B A4B", "vendor": "google", "lab": "Google", "rank": 4, "score": 69.8, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemma 4 E4B", "vendor": "google", "lab": "Google", "rank": 5, "score": 69.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Olmo 3 32B", "vendor": "other", "lab": "Allen AI", "rank": 6, "score": 54.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Allen AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "K2-V2", "vendor": "other", "lab": "MBZUAI", "rank": 7, "score": 53.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "MBZUAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemma 4 E2B", "vendor": "google", "lab": "Google", "rank": 8, "score": 53.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}]
},
{
id: "gsm8k", name: "GSM8K", category: "Math", fieldSize: 23,
description: "Math reasoning benchmark from OpenAI (Cobbe et al., 2021). Linguistically diverse grade-school word problems that take between two and eight steps of arithmetic to solve. Scored by exact match on the final numeric answer over the 1,319-problem test set.",
unit: "accuracy %", higherIsBetter: true,
links: { dataset: "https://huggingface.co/datasets/openai/gsm8k", paper: "https://arxiv.org/abs/2110.14168", paperLabel: "Paper (Cobbe et al., 2021)" },
aligned: { rank: 8, score: 93.3, sample: "150 questions", runId: "36316454589" },
competitors: [{"model": "Nemotron 3 Nano 30B A3B", "vendor": "other", "lab": "NVIDIA", "rank": 1, "score": 95.4, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "NVIDIA (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek V4 Pro", "vendor": "deepseek", "lab": "DeepSeek", "rank": 2, "score": 95.1, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Kimi K2", "vendor": "other", "lab": "Moonshot AI", "rank": 3, "score": 95.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Moonshot AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-5", "vendor": "openai", "lab": "OpenAI", "rank": 4, "score": 94.8, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "INTELLECT-3", "vendor": "other", "lab": "Prime Intellect", "rank": 4, "score": 94.8, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Prime Intellect (Aligned competitor dataset, Sep 28 2026)"}, {"model": "MiniMax M2.1", "vendor": "other", "lab": "MiniMax", "rank": 6, "score": 94.6, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "MiniMax (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Qwen3 Coder 30B A3B Instruct", "vendor": "qwen", "lab": "Alibaba", "rank": 7, "score": 94.2, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Alibaba (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek-V4-Pro (base)", "vendor": "deepseek", "lab": "DeepSeek", "rank": 9, "score": 92.6, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Nemotron 3 Nano 30B-A3B (base)", "vendor": "other", "lab": "NVIDIA", "rank": 10, "score": 92.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "NVIDIA (Aligned competitor dataset, Sep 28 2026)"}, {"model": "MiMo-V2-Flash (base)", "vendor": "other", "lab": "Xiaomi", "rank": 10, "score": 92.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Xiaomi (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Kimi-K2 (base)", "vendor": "other", "lab": "Moonshot AI", "rank": 12, "score": 92.1, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Moonshot AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek-V3.1 (base)", "vendor": "deepseek", "lab": "DeepSeek", "rank": 13, "score": 91.4, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek-V3.2 (base)", "vendor": "deepseek", "lab": "DeepSeek", "rank": 14, "score": 91.1, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek-V4-Flash (base)", "vendor": "deepseek", "lab": "DeepSeek", "rank": 15, "score": 90.8, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Seed-OSS-36B (base)", "vendor": "other", "lab": "ByteDance", "rank": 15, "score": 90.8, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "ByteDance (Aligned competitor dataset, Sep 28 2026)"}]
},
{
id: "arc-challenge", name: "ARC-Challenge", category: "Knowledge", fieldSize: 20,
description: "Science reasoning benchmark from the Allen Institute for AI (Clark et al., 2018). Multiple-choice questions from grade 3 to 9 science exams. The Challenge set contains only questions that both a retrieval-based and a word co-occurrence baseline answered incorrectly.",
unit: "accuracy %", higherIsBetter: true,
links: { dataset: "https://huggingface.co/datasets/allenai/ai2_arc", paper: "https://arxiv.org/abs/1803.05457", paperLabel: "Paper (Clark et al., 2018)" },
aligned: { rank: 16, score: 93.3, sample: "150 of 1,172 questions", runId: "36425671251" },
competitors: [{"model": "MiMo-V2.5-Pro (base)", "vendor": "other", "lab": "Xiaomi", "rank": 1, "score": 97.2, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Xiaomi (Aligned competitor dataset, Sep 28 2026)"}, {"model": "MiMo-V2.5 (base)", "vendor": "other", "lab": "Xiaomi", "rank": 2, "score": 96.5, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Xiaomi (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-5", "vendor": "openai", "lab": "OpenAI", "rank": 3, "score": 96.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Nemotron 3 Super 120B-A12B (base)", "vendor": "other", "lab": "NVIDIA", "rank": 4, "score": 96.1, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "NVIDIA (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GLM-5", "vendor": "other", "lab": "Zhipu AI", "rank": 5, "score": 96.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Zhipu AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "MiMo-V2-Flash (base)", "vendor": "other", "lab": "Xiaomi", "rank": 6, "score": 95.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Xiaomi (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Kimi-K2 (base)", "vendor": "other", "lab": "Moonshot AI", "rank": 7, "score": 95.7, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Moonshot AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek-V3.1 (base)", "vendor": "deepseek", "lab": "DeepSeek", "rank": 8, "score": 95.6, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek-V3.2-Exp (base)", "vendor": "deepseek", "lab": "DeepSeek", "rank": 9, "score": 95.5, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek V4 Flash", "vendor": "deepseek", "lab": "DeepSeek", "rank": 10, "score": 95.4, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Qwen3 235B A22B Instruct 2507", "vendor": "qwen", "lab": "Alibaba", "rank": 11, "score": 94.8, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Alibaba (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Ling-flash-base-2.0 (base)", "vendor": "other", "lab": "inclusionAI (Ant Group)", "rank": 11, "score": 94.8, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "inclusionAI (Ant Group) (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Mistral Medium 3.1", "vendor": "mistral", "lab": "Mistral AI", "rank": 13, "score": 93.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Mistral AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GLM-4.5-Air (base)", "vendor": "other", "lab": "Zhipu AI", "rank": 13, "score": 93.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Zhipu AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Nemotron 3 Nano Omni", "vendor": "other", "lab": "NVIDIA", "rank": 15, "score": 93.5, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "NVIDIA (Aligned competitor dataset, Sep 28 2026)"}]
},
{
id: "triviaqa", name: "TriviaQA", category: "Knowledge", fieldSize: 17,
description: "Open-domain question answering benchmark from the University of Washington (Joshi et al., 2017). Questions authored by trivia enthusiasts, answered here without supplied evidence documents. Scored by exact match against the accepted answer aliases.",
unit: "accuracy %", higherIsBetter: true,
links: { dataset: "https://huggingface.co/datasets/mandarjoshi/trivia_qa", paper: "https://arxiv.org/abs/1705.03551", paperLabel: "Paper (Joshi et al., 2017)" },
aligned: { rank: 8, score: 81.6, sample: "", runId: "" },
competitors: [{"model": "DeepSeek-V4-Pro (base)", "vendor": "deepseek", "lab": "DeepSeek", "rank": 1, "score": 85.6, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Kimi K2 (base)", "vendor": "other", "lab": "Moonshot AI", "rank": 2, "score": 85.1, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Moonshot AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek-V3.2-Exp (base)", "vendor": "deepseek", "lab": "DeepSeek", "rank": 3, "score": 83.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek-V3.1 (base)", "vendor": "deepseek", "lab": "DeepSeek", "rank": 4, "score": 83.5, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek-V3.2 (base)", "vendor": "deepseek", "lab": "DeepSeek", "rank": 5, "score": 83.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek-V4-Flash (base)", "vendor": "deepseek", "lab": "DeepSeek", "rank": 6, "score": 82.8, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Seed-OSS-36B (base)", "vendor": "other", "lab": "ByteDance", "rank": 7, "score": 82.1, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "ByteDance (Aligned competitor dataset, Sep 28 2026)"}, {"model": "MiMo-V2.5-Pro (base)", "vendor": "other", "lab": "Xiaomi", "rank": 9, "score": 81.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Xiaomi (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GLM-5 (base)", "vendor": "other", "lab": "Zhipu AI", "rank": 10, "score": 80.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Zhipu AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "MiMo-V2.5 (base)", "vendor": "other", "lab": "Xiaomi", "rank": 11, "score": 80.7, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Xiaomi (Aligned competitor dataset, Sep 28 2026)"}, {"model": "MiMo-V2-Flash (base)", "vendor": "other", "lab": "Xiaomi", "rank": 12, "score": 80.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Xiaomi (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GLM-4.5 (base)", "vendor": "other", "lab": "Zhipu AI", "rank": 13, "score": 80.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Zhipu AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Kimi Linear 48B-A3B (base)", "vendor": "other", "lab": "Moonshot AI", "rank": 14, "score": 75.2, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Moonshot AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Ministral 3 14B (base)", "vendor": "mistral", "lab": "Mistral AI", "rank": 15, "score": 74.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Mistral AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Ministral 3 8B (base)", "vendor": "mistral", "lab": "Mistral AI", "rank": 16, "score": 68.1, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Mistral AI (Aligned competitor dataset, Sep 28 2026)"}]
},
{
id: "humaneval", name: "HumanEval", category: "Code", fieldSize: 19,
description: "Code-generation benchmark from OpenAI (Chen et al., 2021). 164 hand-written Python programming problems, each with a function signature, docstring, and unit tests. A solution counts as correct only if it passes every test.",
unit: "accuracy %", higherIsBetter: true,
links: { dataset: "https://huggingface.co/datasets/openai/openai_humaneval", datasetLabel: "Official dataset (OpenAI)", paper: "https://arxiv.org/abs/2107.03374", paperLabel: "Paper (Chen et al., 2021)" },
aligned: { rank: 16, score: 92.0, sample: "138 of 150 problems passed", runId: "36425671251" },
competitors: [{"model": "Claude Sonnet 4.5", "vendor": "anthropic", "lab": "Anthropic", "rank": 1, "score": 97.6, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Anthropic (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Grok 4", "vendor": "xai", "lab": "xAI", "rank": 2, "score": 97.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "xAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GLM-5", "vendor": "other", "lab": "Zhipu AI", "rank": 2, "score": 97.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Zhipu AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Claude Opus 4.6", "vendor": "anthropic", "lab": "Anthropic", "rank": 2, "score": 97.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Anthropic (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Claude Opus 4.5", "vendor": "anthropic", "lab": "Anthropic", "rank": 2, "score": 97.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Anthropic (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemini 3 Pro", "vendor": "google", "lab": "Google", "rank": 2, "score": 97.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Claude Haiku 4.5", "vendor": "anthropic", "lab": "Anthropic", "rank": 7, "score": 96.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Anthropic (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Qwen3.5 397B A17B", "vendor": "qwen", "lab": "Alibaba", "rank": 7, "score": 96.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Alibaba (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Claude Opus 4.1", "vendor": "anthropic", "lab": "Anthropic", "rank": 9, "score": 95.7, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Anthropic (Aligned competitor dataset, Sep 28 2026)"}, {"model": "INTELLECT-3", "vendor": "other", "lab": "Prime Intellect", "rank": 10, "score": 95.1, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Prime Intellect (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-5.1", "vendor": "openai", "lab": "OpenAI", "rank": 11, "score": 94.5, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek V3.2", "vendor": "deepseek", "lab": "DeepSeek", "rank": 12, "score": 93.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Nemotron 3 Nano 30B A3B", "vendor": "other", "lab": "NVIDIA", "rank": 13, "score": 93.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "NVIDIA (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Qwen3.5 Plus", "vendor": "qwen", "lab": "Alibaba", "rank": 13, "score": 93.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Alibaba (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Qwen3 Coder 480B A35B", "vendor": "qwen", "lab": "Alibaba", "rank": 15, "score": 92.7, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Alibaba (Aligned competitor dataset, Sep 28 2026)"}]
},
{
id: "mbpp", name: "MBPP", category: "Code", fieldSize: 16,
description: "Mostly Basic Python Problems, a code-generation benchmark from Google Research (Austin et al., 2021). Entry-level Python programming tasks, each with a short description and three test cases. A solution counts as correct only if it passes every test.",
unit: "accuracy %", higherIsBetter: true,
links: { dataset: "https://huggingface.co/datasets/google-research-datasets/mbpp", datasetLabel: "Official dataset (Google Research)", paper: "https://arxiv.org/abs/2108.07732", paperLabel: "Paper (Austin et al., 2021)" },
aligned: { rank: 4, score: 84.0, sample: "", runId: "" },
competitors: [{"model": "Sarvam-30B", "vendor": "other", "lab": "Sarvam AI", "rank": 1, "score": 92.7, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Sarvam AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "MiniCPM-SALA", "vendor": "other", "lab": "OpenBMB", "rank": 2, "score": 89.1, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenBMB (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemini 3 Pro", "vendor": "google", "lab": "Google", "rank": 3, "score": 86.2, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Stable-DiffCoder-8B (base)", "vendor": "other", "lab": "ByteDance", "rank": 5, "score": 83.6, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "ByteDance (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-5 (high)", "vendor": "openai", "lab": "OpenAI", "rank": 6, "score": 83.1, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "ERNIE 5.0", "vendor": "other", "lab": "Baidu", "rank": 7, "score": 82.5, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Baidu (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Kimi K2 Instruct", "vendor": "other", "lab": "Moonshot AI", "rank": 8, "score": 81.8, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Moonshot AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek-V3.2 (thinking)", "vendor": "deepseek", "lab": "DeepSeek", "rank": 9, "score": 81.5, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Seed-OSS-36B (base)", "vendor": "other", "lab": "ByteDance", "rank": 10, "score": 80.6, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "ByteDance (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Qwen3-235B-A22B-Instruct-2507", "vendor": "qwen", "lab": "Alibaba", "rank": 11, "score": 79.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Alibaba (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek-V3.1", "vendor": "deepseek", "lab": "DeepSeek", "rank": 11, "score": 79.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "LongCat-Flash-Chat", "vendor": "other", "lab": "Meituan", "rank": 13, "score": 79.6, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Meituan (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Kimi K2 (base)", "vendor": "other", "lab": "Moonshot AI", "rank": 14, "score": 79.4, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Moonshot AI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "ERNIE 5.0 (base)", "vendor": "other", "lab": "Baidu", "rank": 15, "score": 79.1, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Baidu (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek-V3.2-Exp (base)", "vendor": "deepseek", "lab": "DeepSeek", "rank": 16, "score": 78.8, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}]
},
{
id: "math-500", name: "MATH-500", category: "Math", fieldSize: 114,
description: "Competition mathematics benchmark. A 500-problem subset, selected by OpenAI (Lightman et al., 2023), of the MATH dataset of high-school competition problems (Hendrycks et al., 2021). Scored on the final boxed answer.",
unit: "accuracy %", higherIsBetter: true,
links: { dataset: "https://huggingface.co/datasets/HuggingFaceH4/MATH-500", paper: "https://arxiv.org/abs/2103.03874", paperLabel: "Paper (Hendrycks et al., 2021)", extra: [{ label: "Subset paper (Lightman et al., 2023)", url: "https://arxiv.org/abs/2305.20050" }] },
aligned: { rank: 16, score: 96.6, sample: "144 of 149 graded questions (150 of 500 sampled)", runId: "36425671251" },
competitors: [{"model": "Qwen3 235B A22B Thinking 2507", "vendor": "qwen", "lab": "Alibaba", "rank": 1, "score": 99.6, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Alibaba (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GPT-5", "vendor": "openai", "lab": "OpenAI", "rank": 2, "score": 99.4, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "OpenAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "LongCat-Flash-Thinking", "vendor": "other", "lab": "Meituan", "rank": 3, "score": 99.2, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Meituan (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemini 3.8 Flash", "vendor": "google", "lab": "Google", "rank": 4, "score": 99.1, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemini 3.7 Flash", "vendor": "google", "lab": "Google", "rank": 5, "score": 99.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Grok 4", "vendor": "xai", "lab": "xAI", "rank": 5, "score": 99.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "xAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek V3.1", "vendor": "deepseek", "lab": "DeepSeek", "rank": 7, "score": 98.8, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Llama 3.3 Nemotron Super 49B v1.5", "vendor": "other", "lab": "NVIDIA", "rank": 8, "score": 98.3, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "NVIDIA (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Grok 4.6", "vendor": "xai", "lab": "xAI", "rank": 9, "score": 98.2, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "xAI (Aligned competitor dataset, Sep 28 2026)"}, {"model": "DeepSeek V4 Pro 0813", "vendor": "deepseek", "lab": "DeepSeek", "rank": 9, "score": 98.2, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "DeepSeek (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Qwen3 235B A22B Instruct 2507", "vendor": "qwen", "lab": "Alibaba", "rank": 11, "score": 98.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Alibaba (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Qwen3.5 Flash", "vendor": "qwen", "lab": "Alibaba", "rank": 11, "score": 98.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Alibaba (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Qwen3.6 Max Preview", "vendor": "qwen", "lab": "Alibaba", "rank": 11, "score": 98.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Alibaba (Aligned competitor dataset, Sep 28 2026)"}, {"model": "Gemini 3.6 Flash", "vendor": "google", "lab": "Google", "rank": 11, "score": 98.0, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Google (Aligned competitor dataset, Sep 28 2026)"}, {"model": "GLM-4.5", "vendor": "other", "lab": "Zhipu AI", "rank": 15, "score": 97.9, "source": "aligned-competitor-data-2026-09-28", "sourceLabel": "Zhipu AI (Aligned competitor dataset, Sep 28 2026)"}]
}
]
};
/* ---------------------------------------------------------------------------
* DISABLED: cost / latency boards (provenance unverified). Not loaded.
* Kept here verbatim from the previous page's static tables so they can be
* restored if Chris confirms sources. To restore: move entries into
* BENCH_DATA.boards with category "Costs" (add a Costs color: dot
* "oklch(62.7% .194 149.214)", tagBg "oklch(96.2% .044 156.743)",
* tagText "oklch(39.3% .095 152.535)", tagRing "oklch(87.1% .15 154.449 / .7)")
* and higherIsBetter:false. The renderer would also need a table/lower-is-better variant.
* The old "Cost vs quality" scatter used a formula-derived quality axis and is
* intentionally NOT kept. Faith boards (CEFEAI) were removed entirely at Chris's request.
*
* pricing (USD per 1M tokens in/out, $/chat, tokens/response):
* Aligned AI Fast 0.05 / 0.08 0.003 420
* Gemini 2.5 Flash Lite 0.10 / 0.40 0.035 54042
* GPT-4o Mini 0.15 / 0.60 0.045 610
* Grok 3 Mini 0.30 / 0.50 0.055 890
* Gemini 2.5 Flash 0.10 / 0.40 0.075 9885
* GPT-5 Mini 0.25 / 2.00 0.150 1200
* o3-mini 1.10 / 4.40 0.180 1800
* Claude Opus 4.7 5.00 / 25.00 0.120 7481
* Gemini 2.5 Pro 1.25 / 10.00 0.210 9808
* Gemini 3.1 Pro 2.00 / 12.00 0.290 7523
* Claude Opus 4.1 15.00 / 75.00 0.320 9200
* Grok 3 3.00 / 15.00 0.340 6400
* GPT-5.2 1.75 / 14.00 0.500 1584
* Claude Opus 4.8 5.50 / 27.50 0.140 6900
* DeepSeek V3 0.27 / 1.10 0.040 5200
* Mistral Large 2.00 / 6.00 0.110 4100
*
* time to first token (ms, median streaming):
* Aligned AI 180 · Grok 3 Mini 195 · Gemini 2.5 Flash Lite 210 · GPT-4o Mini 240 ·
* Gemini 2.5 Flash 280 · GPT-5 Mini 310 · DeepSeek V3 350 · Mistral Large 390 ·
* Claude Opus 4.7 420 · Claude Opus 4.8 450 · Gemini 2.5 Pro 480 · GPT-5.2 510 ·
* Grok 3 560 · Gemini 3.1 Pro 620 · Claude Opus 4.1 740 · o3-mini 890
* ------------------------------------------------------------------------- */
|