| { |
| "generatedAt": "2026-07-25", |
| "benchmarks": { |
| "robloxqa": { |
| "name": "RobloxQA v2.0", |
| "dataset": "https://huggingface.co/datasets/TorpedoSoftware/RobloxQA-v2.0", |
| "description": "Knowledge testing. Question answering about Roblox and Luau.", |
| "problemCounts": { |
| "multipleChoice": 3000, |
| "freeform": 3000 |
| }, |
| "topics": [ |
| { "key": "classes", "label": "Engine classes", "n": 1157 }, |
| { "key": "types", "label": "Enums & types", "n": 320 }, |
| { "key": "luau", "label": "Luau", "n": 87 }, |
| { "key": "scripting", "label": "Scripting & data", "n": 281 }, |
| { "key": "gameplay", "label": "Gameplay systems", "n": 127 }, |
| { "key": "visuals", "label": "Visuals & audio", "n": 116 }, |
| { "key": "art", "label": "Art & avatars", "n": 162 }, |
| { "key": "monetization", "label": "Monetization", "n": 198 }, |
| { "key": "studio", "label": "Studio & tools", "n": 166 }, |
| { "key": "learning", "label": "Tutorials", "n": 386 } |
| ] |
| }, |
| "leetcode": { |
| "name": "Luau Leetcode", |
| "dataset": "https://huggingface.co/datasets/TorpedoSoftware/LuauLeetcode", |
| "description": "Single shot code generation. Leetcode problems in Luau.", |
| "problemCounts": { |
| "easy": 47, |
| "medium": 97, |
| "hard": 64 |
| } |
| } |
| }, |
| "models": [ |
| { |
| "name": "Qwen 2.5 Coder 32B (Q4_K_M)", |
| "provider": "Alibaba", |
| "url": "https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct", |
| "releaseDate": "2024-11-12", |
| "size": 19851335840, |
| "price": { |
| "electricity": 0.2 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 1722, |
| "noAnswer": 0, |
| "inputTokens": 239246, |
| "outputTokens": 157931, |
| "kilowattHour": 0.838 |
| }, |
| "multipleChoice": { |
| "correct": 2537, |
| "noAnswer": 0, |
| "inputTokens": 418468, |
| "outputTokens": 553090, |
| "kilowattHour": 1.808 |
| }, |
| "topics": { |
| "classes": { "freeform": 632, "multipleChoice": 939 }, |
| "types": { "freeform": 168, "multipleChoice": 270 }, |
| "luau": { "freeform": 62, "multipleChoice": 75 }, |
| "scripting": { "freeform": 165, "multipleChoice": 251 }, |
| "gameplay": { "freeform": 50, "multipleChoice": 100 }, |
| "visuals": { "freeform": 62, "multipleChoice": 98 }, |
| "art": { "freeform": 76, "multipleChoice": 133 }, |
| "monetization": { "freeform": 97, "multipleChoice": 153 }, |
| "studio": { "freeform": 107, "multipleChoice": 150 }, |
| "learning": { "freeform": 303, "multipleChoice": 368 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 73.48, |
| "fullPass": 26, |
| "partialPass": 16, |
| "fail": 5, |
| "noCode": 0, |
| "codeQuality": 86.8, |
| "invalidSyntax": 2, |
| "inputTokens": 20709, |
| "outputTokens": 20645, |
| "kilowattHour": 0.08 |
| }, |
| "medium": { |
| "unitTestPassRate": 44.49, |
| "fullPass": 21, |
| "partialPass": 60, |
| "fail": 16, |
| "noCode": 0, |
| "codeQuality": 84.8, |
| "invalidSyntax": 2, |
| "inputTokens": 51013, |
| "outputTokens": 64070, |
| "kilowattHour": 0.248 |
| }, |
| "hard": { |
| "unitTestPassRate": 25.4, |
| "fullPass": 4, |
| "partialPass": 37, |
| "fail": 23, |
| "noCode": 0, |
| "codeQuality": 76.2, |
| "invalidSyntax": 3, |
| "inputTokens": 32784, |
| "outputTokens": 43817, |
| "kilowattHour": 0.169 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Qwen 3 4B Instruct 2507 (Q8_0)", |
| "provider": "Alibaba", |
| "url": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507", |
| "releaseDate": "2025-08-06", |
| "size": 4280405600, |
| "price": { |
| "electricity": 0.2 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 1401, |
| "noAnswer": 0, |
| "inputTokens": 239246, |
| "outputTokens": 182527, |
| "kilowattHour": 0.337 |
| }, |
| "multipleChoice": { |
| "correct": 2464, |
| "noAnswer": 0, |
| "inputTokens": 418468, |
| "outputTokens": 754052, |
| "kilowattHour": 0.624 |
| }, |
| "topics": { |
| "classes": { "freeform": 509, "multipleChoice": 923 }, |
| "types": { "freeform": 151, "multipleChoice": 263 }, |
| "luau": { "freeform": 48, "multipleChoice": 72 }, |
| "scripting": { "freeform": 143, "multipleChoice": 245 }, |
| "gameplay": { "freeform": 34, "multipleChoice": 88 }, |
| "visuals": { "freeform": 45, "multipleChoice": 89 }, |
| "art": { "freeform": 54, "multipleChoice": 132 }, |
| "monetization": { "freeform": 75, "multipleChoice": 154 }, |
| "studio": { "freeform": 79, "multipleChoice": 143 }, |
| "learning": { "freeform": 263, "multipleChoice": 355 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 52.08, |
| "fullPass": 18, |
| "partialPass": 16, |
| "fail": 13, |
| "noCode": 0, |
| "codeQuality": 85.7, |
| "invalidSyntax": 3, |
| "inputTokens": 20709, |
| "outputTokens": 23992, |
| "kilowattHour": 0.021 |
| }, |
| "medium": { |
| "unitTestPassRate": 25.41, |
| "fullPass": 8, |
| "partialPass": 38, |
| "fail": 51, |
| "noCode": 0, |
| "codeQuality": 66.0, |
| "invalidSyntax": 18, |
| "inputTokens": 51013, |
| "outputTokens": 149168, |
| "kilowattHour": 0.132 |
| }, |
| "hard": { |
| "unitTestPassRate": 17.34, |
| "fullPass": 6, |
| "partialPass": 22, |
| "fail": 36, |
| "noCode": 0, |
| "codeQuality": 61.5, |
| "invalidSyntax": 13, |
| "inputTokens": 32784, |
| "outputTokens": 119708, |
| "kilowattHour": 0.106 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Qwen 3 Coder 30B-A3B", |
| "provider": "Alibaba", |
| "url": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct", |
| "releaseDate": "2025-07-31", |
| "size": 61064245248, |
| "price": { |
| "input": 0.07, |
| "output": 1.43 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 1705, |
| "noAnswer": 0, |
| "inputTokens": 239246, |
| "outputTokens": 362822 |
| }, |
| "multipleChoice": { |
| "correct": 2551, |
| "noAnswer": 0, |
| "inputTokens": 418468, |
| "outputTokens": 789209 |
| }, |
| "topics": { |
| "classes": { "freeform": 613, "multipleChoice": 945 }, |
| "types": { "freeform": 184, "multipleChoice": 272 }, |
| "luau": { "freeform": 65, "multipleChoice": 80 }, |
| "scripting": { "freeform": 167, "multipleChoice": 247 }, |
| "gameplay": { "freeform": 54, "multipleChoice": 107 }, |
| "visuals": { "freeform": 57, "multipleChoice": 95 }, |
| "art": { "freeform": 71, "multipleChoice": 131 }, |
| "monetization": { "freeform": 92, "multipleChoice": 161 }, |
| "studio": { "freeform": 109, "multipleChoice": 148 }, |
| "learning": { "freeform": 293, "multipleChoice": 365 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 90.08, |
| "fullPass": 38, |
| "partialPass": 7, |
| "fail": 1, |
| "noCode": 1, |
| "codeQuality": 89.9, |
| "invalidSyntax": 1, |
| "inputTokens": 20709, |
| "outputTokens": 102566 |
| }, |
| "medium": { |
| "unitTestPassRate": 59.02, |
| "fullPass": 34, |
| "partialPass": 44, |
| "fail": 17, |
| "noCode": 2, |
| "codeQuality": 76.0, |
| "invalidSyntax": 4, |
| "inputTokens": 51013, |
| "outputTokens": 226333 |
| }, |
| "hard": { |
| "unitTestPassRate": 41.78, |
| "fullPass": 7, |
| "partialPass": 40, |
| "fail": 17, |
| "noCode": 0, |
| "codeQuality": 72.4, |
| "invalidSyntax": 5, |
| "inputTokens": 32784, |
| "outputTokens": 191231 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Qwen 3.5 9B", |
| "provider": "Alibaba", |
| "url": "https://huggingface.co/Qwen/Qwen3.5-9B", |
| "releaseDate": "2026-03-02", |
| "size": 19306216416, |
| "price": { |
| "input": 0.1, |
| "output": 0.16949 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 1183, |
| "noAnswer": 1217, |
| "inputTokens": 245296, |
| "outputTokens": 79341379 |
| }, |
| "multipleChoice": { |
| "correct": 2235, |
| "noAnswer": 492, |
| "inputTokens": 433181, |
| "outputTokens": 35971658 |
| }, |
| "topics": { |
| "classes": { "freeform": 364, "multipleChoice": 781 }, |
| "types": { "freeform": 125, "multipleChoice": 235 }, |
| "luau": { "freeform": 51, "multipleChoice": 72 }, |
| "scripting": { "freeform": 124, "multipleChoice": 231 }, |
| "gameplay": { "freeform": 36, "multipleChoice": 75 }, |
| "visuals": { "freeform": 37, "multipleChoice": 78 }, |
| "art": { "freeform": 52, "multipleChoice": 121 }, |
| "monetization": { "freeform": 64, "multipleChoice": 148 }, |
| "studio": { "freeform": 80, "multipleChoice": 145 }, |
| "learning": { "freeform": 250, "multipleChoice": 349 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 51.99, |
| "fullPass": 22, |
| "partialPass": 5, |
| "fail": 12, |
| "noCode": 8, |
| "codeQuality": 75.2, |
| "invalidSyntax": 2, |
| "inputTokens": 21828, |
| "outputTokens": 574446 |
| }, |
| "medium": { |
| "unitTestPassRate": 22.46, |
| "fullPass": 16, |
| "partialPass": 10, |
| "fail": 25, |
| "noCode": 46, |
| "codeQuality": 41.6, |
| "invalidSyntax": 6, |
| "inputTokens": 53591, |
| "outputTokens": 3113317 |
| }, |
| "hard": { |
| "unitTestPassRate": 7.2, |
| "fullPass": 2, |
| "partialPass": 6, |
| "fail": 8, |
| "noCode": 48, |
| "codeQuality": 15.7, |
| "invalidSyntax": 4, |
| "inputTokens": 34446, |
| "outputTokens": 3197505 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Qwen 3.6 27B", |
| "provider": "Alibaba", |
| "url": "https://huggingface.co/Qwen/Qwen3.6-27B", |
| "releaseDate": "2026-04-22", |
| "size": 55562855904, |
| "price": { |
| "input": 0.289, |
| "output": 2.7573 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2042, |
| "noAnswer": 5, |
| "inputTokens": 245241, |
| "outputTokens": 3366161 |
| }, |
| "multipleChoice": { |
| "correct": 2697, |
| "noAnswer": 46, |
| "inputTokens": 433158, |
| "outputTokens": 4368786 |
| }, |
| "topics": { |
| "classes": { "freeform": 730, "multipleChoice": 1006 }, |
| "types": { "freeform": 215, "multipleChoice": 287 }, |
| "luau": { "freeform": 67, "multipleChoice": 79 }, |
| "scripting": { "freeform": 206, "multipleChoice": 257 }, |
| "gameplay": { "freeform": 65, "multipleChoice": 104 }, |
| "visuals": { "freeform": 74, "multipleChoice": 104 }, |
| "art": { "freeform": 97, "multipleChoice": 155 }, |
| "monetization": { "freeform": 114, "multipleChoice": 172 }, |
| "studio": { "freeform": 129, "multipleChoice": 160 }, |
| "learning": { "freeform": 345, "multipleChoice": 373 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 90.36, |
| "fullPass": 41, |
| "partialPass": 3, |
| "fail": 2, |
| "noCode": 1, |
| "codeQuality": 88.5, |
| "invalidSyntax": 2, |
| "inputTokens": 21828, |
| "outputTokens": 178968 |
| }, |
| "medium": { |
| "unitTestPassRate": 69.13, |
| "fullPass": 49, |
| "partialPass": 22, |
| "fail": 17, |
| "noCode": 9, |
| "codeQuality": 72.9, |
| "invalidSyntax": 7, |
| "inputTokens": 53591, |
| "outputTokens": 1010606 |
| }, |
| "hard": { |
| "unitTestPassRate": 46.29, |
| "fullPass": 17, |
| "partialPass": 17, |
| "fail": 17, |
| "noCode": 13, |
| "codeQuality": 59.5, |
| "invalidSyntax": 6, |
| "inputTokens": 34380, |
| "outputTokens": 911399 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Qwen 3.6 35B-A3B", |
| "provider": "Alibaba", |
| "url": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B", |
| "releaseDate": "2026-04-16", |
| "size": 71903645408, |
| "price": { |
| "input": 0.14, |
| "output": 1.0747 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 1978, |
| "noAnswer": 0, |
| "inputTokens": 245234, |
| "outputTokens": 4083237 |
| }, |
| "multipleChoice": { |
| "correct": 2693, |
| "noAnswer": 1, |
| "inputTokens": 433158, |
| "outputTokens": 3511454 |
| }, |
| "topics": { |
| "classes": { "freeform": 699, "multipleChoice": 1010 }, |
| "types": { "freeform": 204, "multipleChoice": 281 }, |
| "luau": { "freeform": 64, "multipleChoice": 80 }, |
| "scripting": { "freeform": 197, "multipleChoice": 261 }, |
| "gameplay": { "freeform": 72, "multipleChoice": 103 }, |
| "visuals": { "freeform": 72, "multipleChoice": 106 }, |
| "art": { "freeform": 99, "multipleChoice": 148 }, |
| "monetization": { "freeform": 107, "multipleChoice": 170 }, |
| "studio": { "freeform": 126, "multipleChoice": 161 }, |
| "learning": { "freeform": 338, "multipleChoice": 373 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 94.36, |
| "fullPass": 42, |
| "partialPass": 4, |
| "fail": 1, |
| "noCode": 0, |
| "codeQuality": 95.2, |
| "invalidSyntax": 1, |
| "inputTokens": 21828, |
| "outputTokens": 214906 |
| }, |
| "medium": { |
| "unitTestPassRate": 70.06, |
| "fullPass": 50, |
| "partialPass": 24, |
| "fail": 21, |
| "noCode": 2, |
| "codeQuality": 73.9, |
| "invalidSyntax": 14, |
| "inputTokens": 53591, |
| "outputTokens": 1049662 |
| }, |
| "hard": { |
| "unitTestPassRate": 37.3, |
| "fullPass": 12, |
| "partialPass": 25, |
| "fail": 24, |
| "noCode": 3, |
| "codeQuality": 53.7, |
| "invalidSyntax": 18, |
| "inputTokens": 34446, |
| "outputTokens": 1028336 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Claude Fable 5 (Medium)", |
| "provider": "Anthropic", |
| "url": "https://www.anthropic.com/claude/fable", |
| "releaseDate": "2026-06-09", |
| "size": null, |
| "price": { |
| "input": 10, |
| "output": 50 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2615, |
| "noAnswer": 34, |
| "inputTokens": 322087, |
| "outputTokens": 419423 |
| }, |
| "multipleChoice": { |
| "correct": 2909, |
| "noAnswer": 27, |
| "inputTokens": 594072, |
| "outputTokens": 185583 |
| }, |
| "topics": { |
| "classes": { "freeform": 1012, "multipleChoice": 1123 }, |
| "types": { "freeform": 278, "multipleChoice": 312 }, |
| "luau": { "freeform": 79, "multipleChoice": 86 }, |
| "scripting": { "freeform": 237, "multipleChoice": 268 }, |
| "gameplay": { "freeform": 106, "multipleChoice": 122 }, |
| "visuals": { "freeform": 103, "multipleChoice": 113 }, |
| "art": { "freeform": 133, "multipleChoice": 155 }, |
| "monetization": { "freeform": 154, "multipleChoice": 188 }, |
| "studio": { "freeform": 146, "multipleChoice": 159 }, |
| "learning": { "freeform": 367, "multipleChoice": 383 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 97.08, |
| "fullPass": 44, |
| "partialPass": 3, |
| "fail": 0, |
| "noCode": 0, |
| "codeQuality": 99.6, |
| "invalidSyntax": 0, |
| "inputTokens": 27243, |
| "outputTokens": 8570 |
| }, |
| "medium": { |
| "unitTestPassRate": 98.38, |
| "fullPass": 84, |
| "partialPass": 13, |
| "fail": 0, |
| "noCode": 0, |
| "codeQuality": 96.6, |
| "invalidSyntax": 0, |
| "inputTokens": 68243, |
| "outputTokens": 68974 |
| }, |
| "hard": { |
| "unitTestPassRate": 93.05, |
| "fullPass": 49, |
| "partialPass": 13, |
| "fail": 2, |
| "noCode": 0, |
| "codeQuality": 93.6, |
| "invalidSyntax": 0, |
| "inputTokens": 43473, |
| "outputTokens": 150372 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Claude Haiku 4.5 (Extended Thinking)", |
| "provider": "Anthropic", |
| "url": "https://www.anthropic.com/claude/haiku", |
| "releaseDate": "2025-10-15", |
| "size": null, |
| "price": { |
| "input": 1, |
| "output": 5 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2032, |
| "noAnswer": 0, |
| "inputTokens": 335763, |
| "outputTokens": 1044209 |
| }, |
| "multipleChoice": { |
| "correct": 2685, |
| "noAnswer": 0, |
| "inputTokens": 535399, |
| "outputTokens": 2130172 |
| }, |
| "topics": { |
| "classes": { "freeform": 737, "multipleChoice": 1012 }, |
| "types": { "freeform": 205, "multipleChoice": 288 }, |
| "luau": { "freeform": 66, "multipleChoice": 81 }, |
| "scripting": { "freeform": 202, "multipleChoice": 262 }, |
| "gameplay": { "freeform": 68, "multipleChoice": 104 }, |
| "visuals": { "freeform": 78, "multipleChoice": 106 }, |
| "art": { "freeform": 101, "multipleChoice": 146 }, |
| "monetization": { "freeform": 106, "multipleChoice": 156 }, |
| "studio": { "freeform": 124, "multipleChoice": 158 }, |
| "learning": { "freeform": 345, "multipleChoice": 372 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 95.56, |
| "fullPass": 43, |
| "partialPass": 4, |
| "fail": 0, |
| "noCode": 0, |
| "codeQuality": 96.2, |
| "invalidSyntax": 0, |
| "inputTokens": 23707, |
| "outputTokens": 96163 |
| }, |
| "medium": { |
| "unitTestPassRate": 80.77, |
| "fullPass": 54, |
| "partialPass": 32, |
| "fail": 11, |
| "noCode": 0, |
| "codeQuality": 90.3, |
| "invalidSyntax": 3, |
| "inputTokens": 58625, |
| "outputTokens": 317612 |
| }, |
| "hard": { |
| "unitTestPassRate": 64.32, |
| "fullPass": 27, |
| "partialPass": 23, |
| "fail": 14, |
| "noCode": 0, |
| "codeQuality": 88.9, |
| "invalidSyntax": 2, |
| "inputTokens": 37666, |
| "outputTokens": 224090 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Claude Opus 4.8 (Medium)", |
| "provider": "Anthropic", |
| "url": "https://www.anthropic.com/claude/opus", |
| "releaseDate": "2026-05-28", |
| "size": null, |
| "price": { |
| "input": 5, |
| "output": 25 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2486, |
| "noAnswer": 0, |
| "inputTokens": 322040, |
| "outputTokens": 323578 |
| }, |
| "multipleChoice": { |
| "correct": 2900, |
| "noAnswer": 0, |
| "inputTokens": 594072, |
| "outputTokens": 94349 |
| }, |
| "topics": { |
| "classes": { "freeform": 942, "multipleChoice": 1118 }, |
| "types": { "freeform": 262, "multipleChoice": 306 }, |
| "luau": { "freeform": 81, "multipleChoice": 86 }, |
| "scripting": { "freeform": 238, "multipleChoice": 274 }, |
| "gameplay": { "freeform": 101, "multipleChoice": 120 }, |
| "visuals": { "freeform": 94, "multipleChoice": 110 }, |
| "art": { "freeform": 119, "multipleChoice": 157 }, |
| "monetization": { "freeform": 136, "multipleChoice": 184 }, |
| "studio": { "freeform": 143, "multipleChoice": 163 }, |
| "learning": { "freeform": 370, "multipleChoice": 382 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 97.08, |
| "fullPass": 44, |
| "partialPass": 3, |
| "fail": 0, |
| "noCode": 0, |
| "codeQuality": 99.7, |
| "invalidSyntax": 0, |
| "inputTokens": 27243, |
| "outputTokens": 9412 |
| }, |
| "medium": { |
| "unitTestPassRate": 92.5, |
| "fullPass": 75, |
| "partialPass": 17, |
| "fail": 5, |
| "noCode": 0, |
| "codeQuality": 95.8, |
| "invalidSyntax": 2, |
| "inputTokens": 68243, |
| "outputTokens": 131649 |
| }, |
| "hard": { |
| "unitTestPassRate": 88.06, |
| "fullPass": 47, |
| "partialPass": 11, |
| "fail": 6, |
| "noCode": 0, |
| "codeQuality": 94.2, |
| "invalidSyntax": 1, |
| "inputTokens": 47661, |
| "outputTokens": 360836 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Claude Opus 5 (Medium)", |
| "provider": "Anthropic", |
| "url": "https://www.anthropic.com/claude/opus", |
| "releaseDate": "2026-07-24", |
| "size": null, |
| "price": { |
| "input": 5, |
| "output": 25 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2570, |
| "noAnswer": 4, |
| "inputTokens": 322040, |
| "outputTokens": 930410 |
| }, |
| "multipleChoice": { |
| "correct": 2925, |
| "noAnswer": 2, |
| "inputTokens": 594072, |
| "outputTokens": 372368 |
| }, |
| "topics": { |
| "classes": { "freeform": 985, "multipleChoice": 1131 }, |
| "types": { "freeform": 271, "multipleChoice": 310 }, |
| "luau": { "freeform": 79, "multipleChoice": 85 }, |
| "scripting": { "freeform": 241, "multipleChoice": 276 }, |
| "gameplay": { "freeform": 106, "multipleChoice": 121 }, |
| "visuals": { "freeform": 99, "multipleChoice": 114 }, |
| "art": { "freeform": 128, "multipleChoice": 157 }, |
| "monetization": { "freeform": 141, "multipleChoice": 188 }, |
| "studio": { "freeform": 149, "multipleChoice": 162 }, |
| "learning": { "freeform": 371, "multipleChoice": 381 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 97.08, |
| "fullPass": 44, |
| "partialPass": 3, |
| "fail": 0, |
| "noCode": 0, |
| "codeQuality": 99.7, |
| "invalidSyntax": 0, |
| "inputTokens": 27243, |
| "outputTokens": 11107 |
| }, |
| "medium": { |
| "unitTestPassRate": 96.73, |
| "fullPass": 80, |
| "partialPass": 17, |
| "fail": 0, |
| "noCode": 0, |
| "codeQuality": 96.2, |
| "invalidSyntax": 0, |
| "inputTokens": 68243, |
| "outputTokens": 89446 |
| }, |
| "hard": { |
| "unitTestPassRate": 90.86, |
| "fullPass": 47, |
| "partialPass": 13, |
| "fail": 4, |
| "noCode": 0, |
| "codeQuality": 95.3, |
| "invalidSyntax": 0, |
| "inputTokens": 43473, |
| "outputTokens": 166051 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Claude Sonnet 4 (Medium)", |
| "provider": "Anthropic", |
| "url": "https://www.anthropic.com/claude/sonnet", |
| "releaseDate": "2025-05-22", |
| "size": null, |
| "price": { |
| "input": 3, |
| "output": 15 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2190, |
| "noAnswer": 0, |
| "inputTokens": 335763, |
| "outputTokens": 882703 |
| }, |
| "multipleChoice": { |
| "correct": 2770, |
| "noAnswer": 0, |
| "inputTokens": 535399, |
| "outputTokens": 2085669 |
| }, |
| "topics": { |
| "classes": { "freeform": 811, "multipleChoice": 1054 }, |
| "types": { "freeform": 232, "multipleChoice": 288 }, |
| "luau": { "freeform": 72, "multipleChoice": 81 }, |
| "scripting": { "freeform": 214, "multipleChoice": 268 }, |
| "gameplay": { "freeform": 83, "multipleChoice": 111 }, |
| "visuals": { "freeform": 88, "multipleChoice": 107 }, |
| "art": { "freeform": 98, "multipleChoice": 153 }, |
| "monetization": { "freeform": 113, "multipleChoice": 171 }, |
| "studio": { "freeform": 132, "multipleChoice": 160 }, |
| "learning": { "freeform": 347, "multipleChoice": 377 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 91.04, |
| "fullPass": 40, |
| "partialPass": 5, |
| "fail": 2, |
| "noCode": 0, |
| "codeQuality": 89.7, |
| "invalidSyntax": 2, |
| "inputTokens": 23707, |
| "outputTokens": 197077 |
| }, |
| "medium": { |
| "unitTestPassRate": 88.17, |
| "fullPass": 68, |
| "partialPass": 21, |
| "fail": 8, |
| "noCode": 0, |
| "codeQuality": 84.9, |
| "invalidSyntax": 3, |
| "inputTokens": 58625, |
| "outputTokens": 816953 |
| }, |
| "hard": { |
| "unitTestPassRate": 70.8, |
| "fullPass": 34, |
| "partialPass": 17, |
| "fail": 13, |
| "noCode": 0, |
| "codeQuality": 76.6, |
| "invalidSyntax": 5, |
| "inputTokens": 37666, |
| "outputTokens": 698459 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Claude Sonnet 5 (Medium)", |
| "provider": "Anthropic", |
| "url": "https://www.anthropic.com/claude/sonnet", |
| "releaseDate": "2026-06-30", |
| "size": null, |
| "price": { |
| "input": 2, |
| "output": 10 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2409, |
| "noAnswer": 0, |
| "inputTokens": 322040, |
| "outputTokens": 282788 |
| }, |
| "multipleChoice": { |
| "correct": 2819, |
| "noAnswer": 0, |
| "inputTokens": 594072, |
| "outputTokens": 34592 |
| }, |
| "topics": { |
| "classes": { "freeform": 896, "multipleChoice": 1071 }, |
| "types": { "freeform": 260, "multipleChoice": 298 }, |
| "luau": { "freeform": 79, "multipleChoice": 85 }, |
| "scripting": { "freeform": 232, "multipleChoice": 270 }, |
| "gameplay": { "freeform": 88, "multipleChoice": 117 }, |
| "visuals": { "freeform": 89, "multipleChoice": 106 }, |
| "art": { "freeform": 117, "multipleChoice": 154 }, |
| "monetization": { "freeform": 133, "multipleChoice": 179 }, |
| "studio": { "freeform": 147, "multipleChoice": 161 }, |
| "learning": { "freeform": 368, "multipleChoice": 378 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 93.76, |
| "fullPass": 41, |
| "partialPass": 5, |
| "fail": 1, |
| "noCode": 0, |
| "codeQuality": 95.0, |
| "invalidSyntax": 1, |
| "inputTokens": 27243, |
| "outputTokens": 11663 |
| }, |
| "medium": { |
| "unitTestPassRate": 90.44, |
| "fullPass": 69, |
| "partialPass": 25, |
| "fail": 3, |
| "noCode": 0, |
| "codeQuality": 95.2, |
| "invalidSyntax": 1, |
| "inputTokens": 69545, |
| "outputTokens": 212537 |
| }, |
| "hard": { |
| "unitTestPassRate": 77.45, |
| "fullPass": 32, |
| "partialPass": 26, |
| "fail": 6, |
| "noCode": 0, |
| "codeQuality": 92.2, |
| "invalidSyntax": 2, |
| "inputTokens": 49463, |
| "outputTokens": 461804 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Deepseek R1 0528", |
| "provider": "DeepSeek", |
| "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528", |
| "releaseDate": "2025-05-28", |
| "size": 1369062772000, |
| "price": { |
| "input": 0.5, |
| "output": 2.18 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2043, |
| "noAnswer": 0, |
| "inputTokens": 216371, |
| "outputTokens": 1029639 |
| }, |
| "multipleChoice": { |
| "correct": 2676, |
| "noAnswer": 0, |
| "inputTokens": 397218, |
| "outputTokens": 3450188 |
| }, |
| "topics": { |
| "classes": { "freeform": 747, "multipleChoice": 1011 }, |
| "types": { "freeform": 216, "multipleChoice": 284 }, |
| "luau": { "freeform": 66, "multipleChoice": 80 }, |
| "scripting": { "freeform": 199, "multipleChoice": 257 }, |
| "gameplay": { "freeform": 70, "multipleChoice": 103 }, |
| "visuals": { "freeform": 78, "multipleChoice": 104 }, |
| "art": { "freeform": 98, "multipleChoice": 145 }, |
| "monetization": { "freeform": 106, "multipleChoice": 162 }, |
| "studio": { "freeform": 125, "multipleChoice": 160 }, |
| "learning": { "freeform": 338, "multipleChoice": 370 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 92.61, |
| "fullPass": 41, |
| "partialPass": 4, |
| "fail": 1, |
| "noCode": 1, |
| "codeQuality": 84.0, |
| "invalidSyntax": 1, |
| "inputTokens": 19895, |
| "outputTokens": 252818 |
| }, |
| "medium": { |
| "unitTestPassRate": 78.87, |
| "fullPass": 61, |
| "partialPass": 22, |
| "fail": 12, |
| "noCode": 2, |
| "codeQuality": 72.6, |
| "invalidSyntax": 6, |
| "inputTokens": 49614, |
| "outputTokens": 1251919 |
| }, |
| "hard": { |
| "unitTestPassRate": 57.2, |
| "fullPass": 22, |
| "partialPass": 23, |
| "fail": 11, |
| "noCode": 8, |
| "codeQuality": 56.3, |
| "invalidSyntax": 5, |
| "inputTokens": 31754, |
| "outputTokens": 1147793 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Deepseek V4 Flash", |
| "provider": "DeepSeek", |
| "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", |
| "releaseDate": "2026-04-24", |
| "size": 159609485896, |
| "price": { |
| "input": 0.098, |
| "output": 0.24458 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2153, |
| "noAnswer": 0, |
| "inputTokens": 216601, |
| "outputTokens": 603615 |
| }, |
| "multipleChoice": { |
| "correct": 2717, |
| "noAnswer": 0, |
| "inputTokens": 397210, |
| "outputTokens": 1105987 |
| }, |
| "topics": { |
| "classes": { "freeform": 780, "multipleChoice": 1019 }, |
| "types": { "freeform": 231, "multipleChoice": 285 }, |
| "luau": { "freeform": 72, "multipleChoice": 83 }, |
| "scripting": { "freeform": 215, "multipleChoice": 265 }, |
| "gameplay": { "freeform": 75, "multipleChoice": 108 }, |
| "visuals": { "freeform": 79, "multipleChoice": 101 }, |
| "art": { "freeform": 99, "multipleChoice": 149 }, |
| "monetization": { "freeform": 119, "multipleChoice": 173 }, |
| "studio": { "freeform": 130, "multipleChoice": 158 }, |
| "learning": { "freeform": 353, "multipleChoice": 376 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 90.69, |
| "fullPass": 41, |
| "partialPass": 3, |
| "fail": 3, |
| "noCode": 0, |
| "codeQuality": 94.2, |
| "invalidSyntax": 2, |
| "inputTokens": 19950, |
| "outputTokens": 59465 |
| }, |
| "medium": { |
| "unitTestPassRate": 73.97, |
| "fullPass": 58, |
| "partialPass": 19, |
| "fail": 20, |
| "noCode": 0, |
| "codeQuality": 82.0, |
| "invalidSyntax": 9, |
| "inputTokens": 49568, |
| "outputTokens": 539899 |
| }, |
| "hard": { |
| "unitTestPassRate": 42.11, |
| "fullPass": 20, |
| "partialPass": 11, |
| "fail": 32, |
| "noCode": 1, |
| "codeQuality": 65.0, |
| "invalidSyntax": 13, |
| "inputTokens": 31666, |
| "outputTokens": 687885 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Gemini 2.5 Pro (Medium)", |
| "provider": "Google", |
| "url": "https://ai.google.dev/gemini-api/docs/models/gemini-2.5-pro", |
| "releaseDate": "2025-06-17", |
| "size": null, |
| "price": { |
| "input": 1.25, |
| "output": 10 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2178, |
| "noAnswer": 0, |
| "inputTokens": 199629, |
| "outputTokens": 2317472 |
| }, |
| "multipleChoice": { |
| "correct": 2785, |
| "noAnswer": 0, |
| "inputTokens": 394248, |
| "outputTokens": 3995205 |
| }, |
| "topics": { |
| "classes": { "freeform": 804, "multipleChoice": 1048 }, |
| "types": { "freeform": 222, "multipleChoice": 296 }, |
| "luau": { "freeform": 72, "multipleChoice": 82 }, |
| "scripting": { "freeform": 210, "multipleChoice": 264 }, |
| "gameplay": { "freeform": 87, "multipleChoice": 115 }, |
| "visuals": { "freeform": 80, "multipleChoice": 109 }, |
| "art": { "freeform": 99, "multipleChoice": 152 }, |
| "monetization": { "freeform": 122, "multipleChoice": 180 }, |
| "studio": { "freeform": 135, "multipleChoice": 162 }, |
| "learning": { "freeform": 347, "multipleChoice": 377 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 94.95, |
| "fullPass": 43, |
| "partialPass": 3, |
| "fail": 1, |
| "noCode": 0, |
| "codeQuality": 98.9, |
| "invalidSyntax": 0, |
| "inputTokens": 21524, |
| "outputTokens": 232143 |
| }, |
| "medium": { |
| "unitTestPassRate": 87.78, |
| "fullPass": 64, |
| "partialPass": 27, |
| "fail": 6, |
| "noCode": 0, |
| "codeQuality": 88.1, |
| "invalidSyntax": 4, |
| "inputTokens": 53188, |
| "outputTokens": 1065697 |
| }, |
| "hard": { |
| "unitTestPassRate": 67.55, |
| "fullPass": 29, |
| "partialPass": 21, |
| "fail": 14, |
| "noCode": 0, |
| "codeQuality": 78.9, |
| "invalidSyntax": 7, |
| "inputTokens": 34116, |
| "outputTokens": 1135807 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Gemini 3.1 Pro (Medium)", |
| "provider": "Google", |
| "url": "https://ai.google.dev/gemini-api/docs/models/gemini-3.1-pro-preview", |
| "releaseDate": "2026-02-19", |
| "size": null, |
| "price": { |
| "input": 2, |
| "output": 12.01 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2427, |
| "noAnswer": 0, |
| "inputTokens": 199629, |
| "outputTokens": 1610007 |
| }, |
| "multipleChoice": { |
| "correct": 2896, |
| "noAnswer": 5, |
| "inputTokens": 394248, |
| "outputTokens": 5014795 |
| }, |
| "topics": { |
| "classes": { "freeform": 919, "multipleChoice": 1106 }, |
| "types": { "freeform": 250, "multipleChoice": 308 }, |
| "luau": { "freeform": 79, "multipleChoice": 85 }, |
| "scripting": { "freeform": 233, "multipleChoice": 277 }, |
| "gameplay": { "freeform": 99, "multipleChoice": 118 }, |
| "visuals": { "freeform": 93, "multipleChoice": 109 }, |
| "art": { "freeform": 112, "multipleChoice": 156 }, |
| "monetization": { "freeform": 142, "multipleChoice": 190 }, |
| "studio": { "freeform": 141, "multipleChoice": 164 }, |
| "learning": { "freeform": 359, "multipleChoice": 383 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 95.05, |
| "fullPass": 43, |
| "partialPass": 4, |
| "fail": 0, |
| "noCode": 0, |
| "codeQuality": 97.7, |
| "invalidSyntax": 0, |
| "inputTokens": 21524, |
| "outputTokens": 72196 |
| }, |
| "medium": { |
| "unitTestPassRate": 95.7, |
| "fullPass": 79, |
| "partialPass": 17, |
| "fail": 1, |
| "noCode": 0, |
| "codeQuality": 84.2, |
| "invalidSyntax": 1, |
| "inputTokens": 53188, |
| "outputTokens": 366555 |
| }, |
| "hard": { |
| "unitTestPassRate": 93.59, |
| "fullPass": 50, |
| "partialPass": 11, |
| "fail": 3, |
| "noCode": 0, |
| "codeQuality": 79.1, |
| "invalidSyntax": 0, |
| "inputTokens": 34116, |
| "outputTokens": 414988 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Gemma 3 27B", |
| "provider": "Google", |
| "url": "https://huggingface.co/google/gemma-3-27b-it", |
| "releaseDate": "2025-03-12", |
| "size": 54864813280, |
| "price": { |
| "input": 0.08, |
| "output": 0.37514 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 1720, |
| "noAnswer": 0, |
| "inputTokens": 225213, |
| "outputTokens": 219150 |
| }, |
| "multipleChoice": { |
| "correct": 2536, |
| "noAnswer": 0, |
| "inputTokens": 420584, |
| "outputTokens": 740270 |
| }, |
| "topics": { |
| "classes": { "freeform": 594, "multipleChoice": 946 }, |
| "types": { "freeform": 175, "multipleChoice": 256 }, |
| "luau": { "freeform": 46, "multipleChoice": 72 }, |
| "scripting": { "freeform": 178, "multipleChoice": 256 }, |
| "gameplay": { "freeform": 59, "multipleChoice": 96 }, |
| "visuals": { "freeform": 61, "multipleChoice": 98 }, |
| "art": { "freeform": 80, "multipleChoice": 138 }, |
| "monetization": { "freeform": 95, "multipleChoice": 159 }, |
| "studio": { "freeform": 112, "multipleChoice": 156 }, |
| "learning": { "freeform": 320, "multipleChoice": 359 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 47.23, |
| "fullPass": 15, |
| "partialPass": 13, |
| "fail": 19, |
| "noCode": 0, |
| "codeQuality": 87.0, |
| "invalidSyntax": 4, |
| "inputTokens": 21900, |
| "outputTokens": 9501 |
| }, |
| "medium": { |
| "unitTestPassRate": 25.38, |
| "fullPass": 13, |
| "partialPass": 30, |
| "fail": 54, |
| "noCode": 0, |
| "codeQuality": 65.7, |
| "invalidSyntax": 24, |
| "inputTokens": 53968, |
| "outputTokens": 36512 |
| }, |
| "hard": { |
| "unitTestPassRate": 20.52, |
| "fullPass": 6, |
| "partialPass": 20, |
| "fail": 38, |
| "noCode": 0, |
| "codeQuality": 58.6, |
| "invalidSyntax": 22, |
| "inputTokens": 34628, |
| "outputTokens": 24815 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Gemma 4 26B-A4B", |
| "provider": "Google", |
| "url": "https://huggingface.co/google/gemma-4-26B-A4B", |
| "releaseDate": "2026-04-02", |
| "size": 51611872412, |
| "price": { |
| "input": 0.12, |
| "output": 0.3882 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 1996, |
| "noAnswer": 0, |
| "inputTokens": 268020, |
| "outputTokens": 157880 |
| }, |
| "multipleChoice": { |
| "correct": 2728, |
| "noAnswer": 0, |
| "inputTokens": 465984, |
| "outputTokens": 1067741 |
| }, |
| "topics": { |
| "classes": { "freeform": 717, "multipleChoice": 1022 }, |
| "types": { "freeform": 201, "multipleChoice": 290 }, |
| "luau": { "freeform": 62, "multipleChoice": 79 }, |
| "scripting": { "freeform": 200, "multipleChoice": 261 }, |
| "gameplay": { "freeform": 80, "multipleChoice": 114 }, |
| "visuals": { "freeform": 78, "multipleChoice": 107 }, |
| "art": { "freeform": 85, "multipleChoice": 148 }, |
| "monetization": { "freeform": 100, "multipleChoice": 170 }, |
| "studio": { "freeform": 124, "multipleChoice": 158 }, |
| "learning": { "freeform": 349, "multipleChoice": 379 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 61.75, |
| "fullPass": 27, |
| "partialPass": 4, |
| "fail": 16, |
| "noCode": 0, |
| "codeQuality": 73.7, |
| "invalidSyntax": 12, |
| "inputTokens": 22873, |
| "outputTokens": 37516 |
| }, |
| "medium": { |
| "unitTestPassRate": 54.83, |
| "fullPass": 44, |
| "partialPass": 16, |
| "fail": 36, |
| "noCode": 1, |
| "codeQuality": 86.0, |
| "invalidSyntax": 5, |
| "inputTokens": 55225, |
| "outputTokens": 295668 |
| }, |
| "hard": { |
| "unitTestPassRate": 27.63, |
| "fullPass": 10, |
| "partialPass": 14, |
| "fail": 39, |
| "noCode": 1, |
| "codeQuality": 68.6, |
| "invalidSyntax": 9, |
| "inputTokens": 35820, |
| "outputTokens": 351811 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Gemma 4 31B", |
| "provider": "Google", |
| "url": "https://huggingface.co/google/gemma-4-31B", |
| "releaseDate": "2026-04-02", |
| "size": 62546177752, |
| "price": { |
| "input": 0.14, |
| "output": 0.4256 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2156, |
| "noAnswer": 0, |
| "inputTokens": 280989, |
| "outputTokens": 142754 |
| }, |
| "multipleChoice": { |
| "correct": 2777, |
| "noAnswer": 0, |
| "inputTokens": 471922, |
| "outputTokens": 798907 |
| }, |
| "topics": { |
| "classes": { "freeform": 795, "multipleChoice": 1056 }, |
| "types": { "freeform": 220, "multipleChoice": 292 }, |
| "luau": { "freeform": 66, "multipleChoice": 80 }, |
| "scripting": { "freeform": 210, "multipleChoice": 261 }, |
| "gameplay": { "freeform": 83, "multipleChoice": 114 }, |
| "visuals": { "freeform": 81, "multipleChoice": 106 }, |
| "art": { "freeform": 97, "multipleChoice": 152 }, |
| "monetization": { "freeform": 118, "multipleChoice": 174 }, |
| "studio": { "freeform": 129, "multipleChoice": 161 }, |
| "learning": { "freeform": 357, "multipleChoice": 381 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 82.99, |
| "fullPass": 37, |
| "partialPass": 5, |
| "fail": 5, |
| "noCode": 0, |
| "codeQuality": 92.0, |
| "invalidSyntax": 3, |
| "inputTokens": 22895, |
| "outputTokens": 20959 |
| }, |
| "medium": { |
| "unitTestPassRate": 69.17, |
| "fullPass": 52, |
| "partialPass": 24, |
| "fail": 21, |
| "noCode": 0, |
| "codeQuality": 80.9, |
| "invalidSyntax": 9, |
| "inputTokens": 56157, |
| "outputTokens": 88193 |
| }, |
| "hard": { |
| "unitTestPassRate": 61.61, |
| "fullPass": 26, |
| "partialPass": 24, |
| "fail": 14, |
| "noCode": 0, |
| "codeQuality": 71.6, |
| "invalidSyntax": 5, |
| "inputTokens": 35864, |
| "outputTokens": 101060 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Llama 3.1 8B Instruct", |
| "provider": "Meta", |
| "url": "https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct", |
| "releaseDate": "2024-07-23", |
| "size": 16060522496, |
| "price": { |
| "input": 0.05, |
| "output": 0.071899 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 1443, |
| "noAnswer": 0, |
| "inputTokens": 304283, |
| "outputTokens": 528316 |
| }, |
| "multipleChoice": { |
| "correct": 2362, |
| "noAnswer": 4, |
| "inputTokens": 483150, |
| "outputTokens": 1762943 |
| }, |
| "topics": { |
| "classes": { "freeform": 523, "multipleChoice": 863 }, |
| "types": { "freeform": 158, "multipleChoice": 259 }, |
| "luau": { "freeform": 47, "multipleChoice": 65 }, |
| "scripting": { "freeform": 145, "multipleChoice": 217 }, |
| "gameplay": { "freeform": 36, "multipleChoice": 95 }, |
| "visuals": { "freeform": 40, "multipleChoice": 91 }, |
| "art": { "freeform": 56, "multipleChoice": 133 }, |
| "monetization": { "freeform": 86, "multipleChoice": 142 }, |
| "studio": { "freeform": 93, "multipleChoice": 148 }, |
| "learning": { "freeform": 259, "multipleChoice": 349 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 40.12, |
| "fullPass": 11, |
| "partialPass": 26, |
| "fail": 8, |
| "noCode": 2, |
| "codeQuality": 80.0, |
| "invalidSyntax": 0, |
| "inputTokens": 21190, |
| "outputTokens": 139618 |
| }, |
| "medium": { |
| "unitTestPassRate": 18.94, |
| "fullPass": 4, |
| "partialPass": 52, |
| "fail": 39, |
| "noCode": 2, |
| "codeQuality": 69.6, |
| "invalidSyntax": 6, |
| "inputTokens": 52215, |
| "outputTokens": 163953 |
| }, |
| "hard": { |
| "unitTestPassRate": 11.7, |
| "fullPass": 2, |
| "partialPass": 29, |
| "fail": 32, |
| "noCode": 1, |
| "codeQuality": 69.1, |
| "invalidSyntax": 4, |
| "inputTokens": 33472, |
| "outputTokens": 90364 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Devstral Small 2507 (Q4_K_M)", |
| "provider": "Mistral AI", |
| "url": "https://huggingface.co/mistralai/Devstral-Small-2507", |
| "releaseDate": "2025-07-10", |
| "size": 16089785632, |
| "price": { |
| "electricity": 0.2 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 1652, |
| "noAnswer": 0, |
| "inputTokens": 216225, |
| "outputTokens": 125508, |
| "kilowattHour": 0.352 |
| }, |
| "multipleChoice": { |
| "correct": 2510, |
| "noAnswer": 0, |
| "inputTokens": 398216, |
| "outputTokens": 436601, |
| "kilowattHour": 2.418 |
| }, |
| "topics": { |
| "classes": { "freeform": 587, "multipleChoice": 939 }, |
| "types": { "freeform": 178, "multipleChoice": 263 }, |
| "luau": { "freeform": 50, "multipleChoice": 75 }, |
| "scripting": { "freeform": 171, "multipleChoice": 245 }, |
| "gameplay": { "freeform": 52, "multipleChoice": 96 }, |
| "visuals": { "freeform": 60, "multipleChoice": 94 }, |
| "art": { "freeform": 72, "multipleChoice": 135 }, |
| "monetization": { "freeform": 82, "multipleChoice": 151 }, |
| "studio": { "freeform": 97, "multipleChoice": 153 }, |
| "learning": { "freeform": 303, "multipleChoice": 359 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 65.51, |
| "fullPass": 22, |
| "partialPass": 16, |
| "fail": 9, |
| "noCode": 0, |
| "codeQuality": 90.6, |
| "invalidSyntax": 2, |
| "inputTokens": 20716, |
| "outputTokens": 16228, |
| "kilowattHour": 0.192 |
| }, |
| "medium": { |
| "unitTestPassRate": 34.15, |
| "fullPass": 12, |
| "partialPass": 55, |
| "fail": 30, |
| "noCode": 0, |
| "codeQuality": 84.5, |
| "invalidSyntax": 5, |
| "inputTokens": 51222, |
| "outputTokens": 56048, |
| "kilowattHour": 0.662 |
| }, |
| "hard": { |
| "unitTestPassRate": 21.85, |
| "fullPass": 5, |
| "partialPass": 29, |
| "fail": 30, |
| "noCode": 0, |
| "codeQuality": 80.5, |
| "invalidSyntax": 4, |
| "inputTokens": 32878, |
| "outputTokens": 39727, |
| "kilowattHour": 0.469 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Kimi K3 (High)", |
| "provider": "Moonshot AI", |
| "url": "https://www.kimi.com/blog/kimi-k3", |
| "releaseDate": "2026-07-16", |
| "size": 1560860324864, |
| "price": { |
| "input": 3, |
| "output": 15 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2426, |
| "noAnswer": 0, |
| "inputTokens": 488082, |
| "outputTokens": 1229399 |
| }, |
| "multipleChoice": { |
| "correct": 2894, |
| "noAnswer": 0, |
| "inputTokens": 669758, |
| "outputTokens": 794432 |
| }, |
| "topics": { |
| "classes": { "freeform": 907, "multipleChoice": 1108 }, |
| "types": { "freeform": 258, "multipleChoice": 308 }, |
| "luau": { "freeform": 76, "multipleChoice": 85 }, |
| "scripting": { "freeform": 237, "multipleChoice": 274 }, |
| "gameplay": { "freeform": 93, "multipleChoice": 119 }, |
| "visuals": { "freeform": 97, "multipleChoice": 112 }, |
| "art": { "freeform": 120, "multipleChoice": 156 }, |
| "monetization": { "freeform": 135, "multipleChoice": 184 }, |
| "studio": { "freeform": 140, "multipleChoice": 164 }, |
| "learning": { "freeform": 363, "multipleChoice": 384 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 96.57, |
| "fullPass": 42, |
| "partialPass": 5, |
| "fail": 0, |
| "noCode": 0, |
| "codeQuality": 99.4, |
| "invalidSyntax": 0, |
| "inputTokens": 24300, |
| "outputTokens": 30000 |
| }, |
| "medium": { |
| "unitTestPassRate": 93.85, |
| "fullPass": 75, |
| "partialPass": 20, |
| "fail": 2, |
| "noCode": 0, |
| "codeQuality": 97.1, |
| "invalidSyntax": 0, |
| "inputTokens": 58645, |
| "outputTokens": 298404 |
| }, |
| "hard": { |
| "unitTestPassRate": 91.69, |
| "fullPass": 45, |
| "partialPass": 16, |
| "fail": 3, |
| "noCode": 0, |
| "codeQuality": 92.2, |
| "invalidSyntax": 1, |
| "inputTokens": 37711, |
| "outputTokens": 515974 |
| } |
| } |
| } |
| }, |
| { |
| "name": "GPT 5 (Medium)", |
| "provider": "OpenAI", |
| "url": "https://developers.openai.com/api/docs/models/gpt-5", |
| "releaseDate": "2025-08-07", |
| "size": null, |
| "price": { |
| "input": 1.25, |
| "output": 10 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2364, |
| "noAnswer": 0, |
| "inputTokens": 225589, |
| "outputTokens": 2416184 |
| }, |
| "multipleChoice": { |
| "correct": 2831, |
| "noAnswer": 0, |
| "inputTokens": 404067, |
| "outputTokens": 1886054 |
| }, |
| "topics": { |
| "classes": { "freeform": 886, "multipleChoice": 1085 }, |
| "types": { "freeform": 248, "multipleChoice": 294 }, |
| "luau": { "freeform": 76, "multipleChoice": 81 }, |
| "scripting": { "freeform": 227, "multipleChoice": 273 }, |
| "gameplay": { "freeform": 94, "multipleChoice": 112 }, |
| "visuals": { "freeform": 93, "multipleChoice": 111 }, |
| "art": { "freeform": 121, "multipleChoice": 155 }, |
| "monetization": { "freeform": 127, "multipleChoice": 180 }, |
| "studio": { "freeform": 136, "multipleChoice": 162 }, |
| "learning": { "freeform": 356, "multipleChoice": 378 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 96.99, |
| "fullPass": 44, |
| "partialPass": 3, |
| "fail": 0, |
| "noCode": 0, |
| "codeQuality": 98.4, |
| "invalidSyntax": 0, |
| "inputTokens": 20113, |
| "outputTokens": 83604 |
| }, |
| "medium": { |
| "unitTestPassRate": 96.47, |
| "fullPass": 78, |
| "partialPass": 18, |
| "fail": 1, |
| "noCode": 0, |
| "codeQuality": 92.8, |
| "invalidSyntax": 0, |
| "inputTokens": 49965, |
| "outputTokens": 521703 |
| }, |
| "hard": { |
| "unitTestPassRate": 86.57, |
| "fullPass": 42, |
| "partialPass": 15, |
| "fail": 7, |
| "noCode": 0, |
| "codeQuality": 81.0, |
| "invalidSyntax": 4, |
| "inputTokens": 32013, |
| "outputTokens": 640520 |
| } |
| } |
| } |
| }, |
| { |
| "name": "GPT 5.4 nano (High)", |
| "provider": "OpenAI", |
| "url": "https://developers.openai.com/api/docs/models/gpt-5.4-nano", |
| "releaseDate": "2026-03-17", |
| "size": null, |
| "price": { |
| "input": 0.2, |
| "output": 1.2471 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2183, |
| "noAnswer": 0, |
| "inputTokens": 225589, |
| "outputTokens": 882142 |
| }, |
| "multipleChoice": { |
| "correct": 2670, |
| "noAnswer": 0, |
| "inputTokens": 404080, |
| "outputTokens": 609487 |
| }, |
| "topics": { |
| "classes": { "freeform": 809, "multipleChoice": 1005 }, |
| "types": { "freeform": 231, "multipleChoice": 287 }, |
| "luau": { "freeform": 73, "multipleChoice": 76 }, |
| "scripting": { "freeform": 207, "multipleChoice": 257 }, |
| "gameplay": { "freeform": 77, "multipleChoice": 109 }, |
| "visuals": { "freeform": 78, "multipleChoice": 105 }, |
| "art": { "freeform": 104, "multipleChoice": 146 }, |
| "monetization": { "freeform": 129, "multipleChoice": 159 }, |
| "studio": { "freeform": 129, "multipleChoice": 157 }, |
| "learning": { "freeform": 346, "multipleChoice": 369 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 89.85, |
| "fullPass": 39, |
| "partialPass": 5, |
| "fail": 3, |
| "noCode": 0, |
| "codeQuality": 92.8, |
| "invalidSyntax": 3, |
| "inputTokens": 20935, |
| "outputTokens": 73629 |
| }, |
| "medium": { |
| "unitTestPassRate": 84.93, |
| "fullPass": 64, |
| "partialPass": 25, |
| "fail": 8, |
| "noCode": 0, |
| "codeQuality": 85.9, |
| "invalidSyntax": 6, |
| "inputTokens": 53471, |
| "outputTokens": 488158 |
| }, |
| "hard": { |
| "unitTestPassRate": 65.54, |
| "fullPass": 32, |
| "partialPass": 14, |
| "fail": 16, |
| "noCode": 2, |
| "codeQuality": 69.4, |
| "invalidSyntax": 11, |
| "inputTokens": 38485, |
| "outputTokens": 854382 |
| } |
| } |
| } |
| }, |
| { |
| "name": "GPT 5.5 (Medium)", |
| "provider": "OpenAI", |
| "url": "https://developers.openai.com/api/docs/models/gpt-5.5", |
| "releaseDate": "2026-04-23", |
| "size": null, |
| "price": { |
| "input": 5, |
| "output": 30 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2598, |
| "noAnswer": 0, |
| "inputTokens": 225590, |
| "outputTokens": 881130 |
| }, |
| "multipleChoice": { |
| "correct": 2926, |
| "noAnswer": 0, |
| "inputTokens": 404089, |
| "outputTokens": 550573 |
| }, |
| "topics": { |
| "classes": { "freeform": 1007, "multipleChoice": 1124 }, |
| "types": { "freeform": 279, "multipleChoice": 310 }, |
| "luau": { "freeform": 78, "multipleChoice": 84 }, |
| "scripting": { "freeform": 238, "multipleChoice": 278 }, |
| "gameplay": { "freeform": 107, "multipleChoice": 123 }, |
| "visuals": { "freeform": 104, "multipleChoice": 114 }, |
| "art": { "freeform": 126, "multipleChoice": 157 }, |
| "monetization": { "freeform": 150, "multipleChoice": 191 }, |
| "studio": { "freeform": 141, "multipleChoice": 163 }, |
| "learning": { "freeform": 368, "multipleChoice": 382 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 97.06, |
| "fullPass": 43, |
| "partialPass": 4, |
| "fail": 0, |
| "noCode": 0, |
| "codeQuality": 99.7, |
| "invalidSyntax": 0, |
| "inputTokens": 20113, |
| "outputTokens": 18431 |
| }, |
| "medium": { |
| "unitTestPassRate": 97.44, |
| "fullPass": 82, |
| "partialPass": 14, |
| "fail": 1, |
| "noCode": 0, |
| "codeQuality": 97.9, |
| "invalidSyntax": 0, |
| "inputTokens": 49965, |
| "outputTokens": 154490 |
| }, |
| "hard": { |
| "unitTestPassRate": 93.01, |
| "fullPass": 52, |
| "partialPass": 9, |
| "fail": 3, |
| "noCode": 0, |
| "codeQuality": 97.2, |
| "invalidSyntax": 0, |
| "inputTokens": 32013, |
| "outputTokens": 233033 |
| } |
| } |
| } |
| }, |
| { |
| "name": "GPT 5.6 Luna (Medium)", |
| "provider": "OpenAI", |
| "url": "https://developers.openai.com/api/docs/models/gpt-5.6-luna", |
| "releaseDate": "2026-07-09", |
| "size": null, |
| "price": { |
| "input": 0.2, |
| "output": 1.2 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2397, |
| "noAnswer": 0, |
| "inputTokens": 225589, |
| "outputTokens": 574988 |
| }, |
| "multipleChoice": { |
| "correct": 2873, |
| "noAnswer": 0, |
| "inputTokens": 404067, |
| "outputTokens": 349378 |
| }, |
| "topics": { |
| "classes": { "freeform": 898, "multipleChoice": 1103 }, |
| "types": { "freeform": 259, "multipleChoice": 296 }, |
| "luau": { "freeform": 74, "multipleChoice": 84 }, |
| "scripting": { "freeform": 222, "multipleChoice": 275 }, |
| "gameplay": { "freeform": 100, "multipleChoice": 119 }, |
| "visuals": { "freeform": 90, "multipleChoice": 111 }, |
| "art": { "freeform": 116, "multipleChoice": 157 }, |
| "monetization": { "freeform": 139, "multipleChoice": 180 }, |
| "studio": { "freeform": 138, "multipleChoice": 165 }, |
| "learning": { "freeform": 361, "multipleChoice": 383 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 95.47, |
| "fullPass": 44, |
| "partialPass": 2, |
| "fail": 1, |
| "noCode": 0, |
| "codeQuality": 99.5, |
| "invalidSyntax": 0, |
| "inputTokens": 20113, |
| "outputTokens": 17787 |
| }, |
| "medium": { |
| "unitTestPassRate": 88.98, |
| "fullPass": 74, |
| "partialPass": 16, |
| "fail": 7, |
| "noCode": 0, |
| "codeQuality": 96.6, |
| "invalidSyntax": 1, |
| "inputTokens": 49965, |
| "outputTokens": 117279 |
| }, |
| "hard": { |
| "unitTestPassRate": 75.24, |
| "fullPass": 33, |
| "partialPass": 24, |
| "fail": 7, |
| "noCode": 0, |
| "codeQuality": 88.0, |
| "invalidSyntax": 3, |
| "inputTokens": 32013, |
| "outputTokens": 154252 |
| } |
| } |
| } |
| }, |
| { |
| "name": "GPT 5.6 Sol (Medium)", |
| "provider": "OpenAI", |
| "url": "https://developers.openai.com/api/docs/models/gpt-5.6-sol", |
| "releaseDate": "2026-07-09", |
| "size": null, |
| "price": { |
| "input": 5, |
| "output": 30 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2572, |
| "noAnswer": 0, |
| "inputTokens": 225589, |
| "outputTokens": 581312 |
| }, |
| "multipleChoice": { |
| "correct": 2938, |
| "noAnswer": 0, |
| "inputTokens": 404067, |
| "outputTokens": 208018 |
| }, |
| "topics": { |
| "classes": { "freeform": 988, "multipleChoice": 1135 }, |
| "types": { "freeform": 280, "multipleChoice": 310 }, |
| "luau": { "freeform": 78, "multipleChoice": 85 }, |
| "scripting": { "freeform": 237, "multipleChoice": 277 }, |
| "gameplay": { "freeform": 106, "multipleChoice": 121 }, |
| "visuals": { "freeform": 100, "multipleChoice": 114 }, |
| "art": { "freeform": 129, "multipleChoice": 159 }, |
| "monetization": { "freeform": 149, "multipleChoice": 191 }, |
| "studio": { "freeform": 148, "multipleChoice": 163 }, |
| "learning": { "freeform": 357, "multipleChoice": 383 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 97.0, |
| "fullPass": 42, |
| "partialPass": 5, |
| "fail": 0, |
| "noCode": 0, |
| "codeQuality": 99.8, |
| "invalidSyntax": 0, |
| "inputTokens": 20113, |
| "outputTokens": 10179 |
| }, |
| "medium": { |
| "unitTestPassRate": 98.48, |
| "fullPass": 81, |
| "partialPass": 16, |
| "fail": 0, |
| "noCode": 0, |
| "codeQuality": 98.1, |
| "invalidSyntax": 0, |
| "inputTokens": 50051, |
| "outputTokens": 87592 |
| }, |
| "hard": { |
| "unitTestPassRate": 92.28, |
| "fullPass": 53, |
| "partialPass": 8, |
| "fail": 3, |
| "noCode": 0, |
| "codeQuality": 96.3, |
| "invalidSyntax": 0, |
| "inputTokens": 32013, |
| "outputTokens": 128181 |
| } |
| } |
| } |
| }, |
| { |
| "name": "GPT 5.6 Terra (Medium)", |
| "provider": "OpenAI", |
| "url": "https://developers.openai.com/api/docs/models/gpt-5.6-terra", |
| "releaseDate": "2026-07-09", |
| "size": null, |
| "price": { |
| "input": 2, |
| "output": 12 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2378, |
| "noAnswer": 0, |
| "inputTokens": 225589, |
| "outputTokens": 264017 |
| }, |
| "multipleChoice": { |
| "correct": 2873, |
| "noAnswer": 0, |
| "inputTokens": 404067, |
| "outputTokens": 261108 |
| }, |
| "topics": { |
| "classes": { "freeform": 901, "multipleChoice": 1096 }, |
| "types": { "freeform": 252, "multipleChoice": 304 }, |
| "luau": { "freeform": 76, "multipleChoice": 84 }, |
| "scripting": { "freeform": 225, "multipleChoice": 272 }, |
| "gameplay": { "freeform": 97, "multipleChoice": 117 }, |
| "visuals": { "freeform": 91, "multipleChoice": 107 }, |
| "art": { "freeform": 113, "multipleChoice": 159 }, |
| "monetization": { "freeform": 127, "multipleChoice": 186 }, |
| "studio": { "freeform": 140, "multipleChoice": 165 }, |
| "learning": { "freeform": 356, "multipleChoice": 383 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 97.08, |
| "fullPass": 44, |
| "partialPass": 3, |
| "fail": 0, |
| "noCode": 0, |
| "codeQuality": 99.1, |
| "invalidSyntax": 0, |
| "inputTokens": 20113, |
| "outputTokens": 10682 |
| }, |
| "medium": { |
| "unitTestPassRate": 93.89, |
| "fullPass": 76, |
| "partialPass": 17, |
| "fail": 4, |
| "noCode": 0, |
| "codeQuality": 97.1, |
| "invalidSyntax": 1, |
| "inputTokens": 49965, |
| "outputTokens": 88570 |
| }, |
| "hard": { |
| "unitTestPassRate": 88.55, |
| "fullPass": 49, |
| "partialPass": 9, |
| "fail": 6, |
| "noCode": 0, |
| "codeQuality": 93.2, |
| "invalidSyntax": 2, |
| "inputTokens": 32013, |
| "outputTokens": 125400 |
| } |
| } |
| } |
| }, |
| { |
| "name": "GPT OSS 20B", |
| "provider": "OpenAI", |
| "url": "https://huggingface.co/openai/gpt-oss-20b", |
| "releaseDate": "2025-08-05", |
| "size": 13761264768, |
| "price": { |
| "input": 0.03, |
| "output": 0.15499 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 1977, |
| "noAnswer": 1, |
| "inputTokens": 402625, |
| "outputTokens": 1315332 |
| }, |
| "multipleChoice": { |
| "correct": 2661, |
| "noAnswer": 1, |
| "inputTokens": 581019, |
| "outputTokens": 853656 |
| }, |
| "topics": { |
| "classes": { "freeform": 699, "multipleChoice": 992 }, |
| "types": { "freeform": 212, "multipleChoice": 284 }, |
| "luau": { "freeform": 71, "multipleChoice": 80 }, |
| "scripting": { "freeform": 205, "multipleChoice": 257 }, |
| "gameplay": { "freeform": 64, "multipleChoice": 99 }, |
| "visuals": { "freeform": 72, "multipleChoice": 105 }, |
| "art": { "freeform": 86, "multipleChoice": 144 }, |
| "monetization": { "freeform": 114, "multipleChoice": 169 }, |
| "studio": { "freeform": 123, "multipleChoice": 160 }, |
| "learning": { "freeform": 331, "multipleChoice": 371 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 86.43, |
| "fullPass": 38, |
| "partialPass": 5, |
| "fail": 4, |
| "noCode": 0, |
| "codeQuality": 91.8, |
| "invalidSyntax": 3, |
| "inputTokens": 22886, |
| "outputTokens": 58350 |
| }, |
| "medium": { |
| "unitTestPassRate": 71.03, |
| "fullPass": 54, |
| "partialPass": 24, |
| "fail": 17, |
| "noCode": 2, |
| "codeQuality": 80.8, |
| "invalidSyntax": 10, |
| "inputTokens": 55688, |
| "outputTokens": 502154 |
| }, |
| "hard": { |
| "unitTestPassRate": 41.2, |
| "fullPass": 16, |
| "partialPass": 15, |
| "fail": 28, |
| "noCode": 5, |
| "codeQuality": 55.9, |
| "invalidSyntax": 15, |
| "inputTokens": 35789, |
| "outputTokens": 618574 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Luau Devstral 24B Instruct v0.2 (Q5_K_XL)", |
| "provider": "Torpedo Software", |
| "url": "https://huggingface.co/TorpedoSoftware/Luau-Devstral-24B-Instruct-v0.2", |
| "releaseDate": "2025-09-16", |
| "size": 16788120384, |
| "price": { |
| "electricity": 0.2 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 1921, |
| "noAnswer": 0, |
| "inputTokens": 216225, |
| "outputTokens": 121041, |
| "kilowattHour": 0.42 |
| }, |
| "multipleChoice": { |
| "correct": 2577, |
| "noAnswer": 0, |
| "inputTokens": 398216, |
| "outputTokens": 304109, |
| "kilowattHour": 0.934 |
| }, |
| "topics": { |
| "classes": { "freeform": 687, "multipleChoice": 958 }, |
| "types": { "freeform": 194, "multipleChoice": 271 }, |
| "luau": { "freeform": 57, "multipleChoice": 76 }, |
| "scripting": { "freeform": 196, "multipleChoice": 253 }, |
| "gameplay": { "freeform": 72, "multipleChoice": 99 }, |
| "visuals": { "freeform": 69, "multipleChoice": 95 }, |
| "art": { "freeform": 91, "multipleChoice": 138 }, |
| "monetization": { "freeform": 107, "multipleChoice": 160 }, |
| "studio": { "freeform": 113, "multipleChoice": 155 }, |
| "learning": { "freeform": 335, "multipleChoice": 372 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 72.34, |
| "fullPass": 26, |
| "partialPass": 16, |
| "fail": 5, |
| "noCode": 0, |
| "codeQuality": 96.0, |
| "invalidSyntax": 0, |
| "inputTokens": 20716, |
| "outputTokens": 23893, |
| "kilowattHour": 0.054 |
| }, |
| "medium": { |
| "unitTestPassRate": 36.75, |
| "fullPass": 15, |
| "partialPass": 58, |
| "fail": 24, |
| "noCode": 0, |
| "codeQuality": 95.8, |
| "invalidSyntax": 0, |
| "inputTokens": 51222, |
| "outputTokens": 66540, |
| "kilowattHour": 0.168 |
| }, |
| "hard": { |
| "unitTestPassRate": 31.39, |
| "fullPass": 8, |
| "partialPass": 41, |
| "fail": 15, |
| "noCode": 0, |
| "codeQuality": 93.7, |
| "invalidSyntax": 0, |
| "inputTokens": 32878, |
| "outputTokens": 44250, |
| "kilowattHour": 0.112 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Luau Qwen3 4B FIM v0.1 (Q8_0)", |
| "provider": "Torpedo Software", |
| "url": "https://huggingface.co/TorpedoSoftware/Luau-Qwen3-4B-FIM-v0.1", |
| "releaseDate": "2025-10-19", |
| "size": 4280402464, |
| "price": { |
| "electricity": 0.2 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 1456, |
| "noAnswer": 0, |
| "inputTokens": 239246, |
| "outputTokens": 132998, |
| "kilowattHour": 0.103 |
| }, |
| "multipleChoice": { |
| "correct": 2468, |
| "noAnswer": 0, |
| "inputTokens": 418468, |
| "outputTokens": 500586, |
| "kilowattHour": 0.402 |
| }, |
| "topics": { |
| "classes": { "freeform": 543, "multipleChoice": 917 }, |
| "types": { "freeform": 158, "multipleChoice": 264 }, |
| "luau": { "freeform": 47, "multipleChoice": 75 }, |
| "scripting": { "freeform": 140, "multipleChoice": 236 }, |
| "gameplay": { "freeform": 42, "multipleChoice": 91 }, |
| "visuals": { "freeform": 54, "multipleChoice": 91 }, |
| "art": { "freeform": 54, "multipleChoice": 129 }, |
| "monetization": { "freeform": 71, "multipleChoice": 158 }, |
| "studio": { "freeform": 85, "multipleChoice": 148 }, |
| "learning": { "freeform": 262, "multipleChoice": 359 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 35.37, |
| "fullPass": 13, |
| "partialPass": 9, |
| "fail": 9, |
| "noCode": 16, |
| "codeQuality": 45.9, |
| "invalidSyntax": 5, |
| "inputTokens": 20709, |
| "outputTokens": 34978, |
| "kilowattHour": 0.08 |
| }, |
| "medium": { |
| "unitTestPassRate": 14.77, |
| "fullPass": 6, |
| "partialPass": 26, |
| "fail": 27, |
| "noCode": 38, |
| "codeQuality": 35.8, |
| "invalidSyntax": 15, |
| "inputTokens": 51013, |
| "outputTokens": 175066, |
| "kilowattHour": 0.399 |
| }, |
| "hard": { |
| "unitTestPassRate": 9.32, |
| "fullPass": 2, |
| "partialPass": 16, |
| "fail": 21, |
| "noCode": 25, |
| "codeQuality": 31.3, |
| "invalidSyntax": 9, |
| "inputTokens": 32784, |
| "outputTokens": 80208, |
| "kilowattHour": 0.183 |
| } |
| } |
| } |
| }, |
| { |
| "name": "Grok 4.5 (Medium)", |
| "provider": "xAI", |
| "url": "https://docs.x.ai/developers/models/grok-4.5", |
| "releaseDate": "2026-07-08", |
| "size": null, |
| "price": { |
| "input": 2, |
| "output": 6.4831 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2441, |
| "noAnswer": 0, |
| "inputTokens": 827473, |
| "outputTokens": 1723323 |
| }, |
| "multipleChoice": { |
| "correct": 2887, |
| "noAnswer": 0, |
| "inputTokens": 1009787, |
| "outputTokens": 1977283 |
| }, |
| "topics": { |
| "classes": { "freeform": 934, "multipleChoice": 1113 }, |
| "types": { "freeform": 267, "multipleChoice": 300 }, |
| "luau": { "freeform": 81, "multipleChoice": 85 }, |
| "scripting": { "freeform": 230, "multipleChoice": 276 }, |
| "gameplay": { "freeform": 96, "multipleChoice": 119 }, |
| "visuals": { "freeform": 94, "multipleChoice": 113 }, |
| "art": { "freeform": 112, "multipleChoice": 155 }, |
| "monetization": { "freeform": 132, "multipleChoice": 182 }, |
| "studio": { "freeform": 138, "multipleChoice": 163 }, |
| "learning": { "freeform": 357, "multipleChoice": 381 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 97.58, |
| "fullPass": 44, |
| "partialPass": 3, |
| "fail": 0, |
| "noCode": 0, |
| "codeQuality": 99.0, |
| "invalidSyntax": 0, |
| "inputTokens": 29842, |
| "outputTokens": 60988 |
| }, |
| "medium": { |
| "unitTestPassRate": 95.02, |
| "fullPass": 79, |
| "partialPass": 14, |
| "fail": 4, |
| "noCode": 0, |
| "codeQuality": 96.2, |
| "invalidSyntax": 1, |
| "inputTokens": 70220, |
| "outputTokens": 539460 |
| }, |
| "hard": { |
| "unitTestPassRate": 83.88, |
| "fullPass": 40, |
| "partialPass": 17, |
| "fail": 7, |
| "noCode": 0, |
| "codeQuality": 91.5, |
| "invalidSyntax": 2, |
| "inputTokens": 45378, |
| "outputTokens": 824883 |
| } |
| } |
| } |
| }, |
| { |
| "name": "GLM 4.7 Flash", |
| "provider": "Z.ai", |
| "url": "https://huggingface.co/zai-org/GLM-4.7-Flash", |
| "releaseDate": "2026-01-19", |
| "size": 31221488576, |
| "price": { |
| "input": 0.06, |
| "output": 0.4 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 1723, |
| "noAnswer": 12, |
| "inputTokens": 218150, |
| "outputTokens": 2226246 |
| }, |
| "multipleChoice": { |
| "correct": 2494, |
| "noAnswer": 91, |
| "inputTokens": 396863, |
| "outputTokens": 7314366 |
| }, |
| "topics": { |
| "classes": { "freeform": 617, "multipleChoice": 921 }, |
| "types": { "freeform": 182, "multipleChoice": 262 }, |
| "luau": { "freeform": 58, "multipleChoice": 71 }, |
| "scripting": { "freeform": 182, "multipleChoice": 250 }, |
| "gameplay": { "freeform": 52, "multipleChoice": 91 }, |
| "visuals": { "freeform": 52, "multipleChoice": 94 }, |
| "art": { "freeform": 74, "multipleChoice": 130 }, |
| "monetization": { "freeform": 98, "multipleChoice": 154 }, |
| "studio": { "freeform": 98, "multipleChoice": 154 }, |
| "learning": { "freeform": 310, "multipleChoice": 367 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 52.99, |
| "fullPass": 20, |
| "partialPass": 8, |
| "fail": 12, |
| "noCode": 7, |
| "codeQuality": 77.8, |
| "invalidSyntax": 0, |
| "inputTokens": 19911, |
| "outputTokens": 493838 |
| }, |
| "medium": { |
| "unitTestPassRate": 20.44, |
| "fullPass": 11, |
| "partialPass": 19, |
| "fail": 23, |
| "noCode": 44, |
| "codeQuality": 40.9, |
| "invalidSyntax": 5, |
| "inputTokens": 49306, |
| "outputTokens": 2496010 |
| }, |
| "hard": { |
| "unitTestPassRate": 6.75, |
| "fullPass": 0, |
| "partialPass": 9, |
| "fail": 10, |
| "noCode": 45, |
| "codeQuality": 17.8, |
| "invalidSyntax": 4, |
| "inputTokens": 31479, |
| "outputTokens": 2463034 |
| } |
| } |
| } |
| }, |
| { |
| "name": "GLM 5.2 (High)", |
| "provider": "Z.ai", |
| "url": "https://huggingface.co/zai-org/GLM-5.2", |
| "releaseDate": "2026-06-16", |
| "size": 1506659919872, |
| "price": { |
| "input": 0.826, |
| "output": 3.5017 |
| }, |
| "scores": { |
| "robloxqa": { |
| "freeform": { |
| "correct": 2230, |
| "noAnswer": 87, |
| "inputTokens": 239177, |
| "outputTokens": 6894184 |
| }, |
| "multipleChoice": { |
| "correct": 2670, |
| "noAnswer": 180, |
| "inputTokens": 417937, |
| "outputTokens": 12567539 |
| }, |
| "topics": { |
| "classes": { "freeform": 823, "multipleChoice": 997 }, |
| "types": { "freeform": 230, "multipleChoice": 277 }, |
| "luau": { "freeform": 74, "multipleChoice": 80 }, |
| "scripting": { "freeform": 219, "multipleChoice": 257 }, |
| "gameplay": { "freeform": 87, "multipleChoice": 107 }, |
| "visuals": { "freeform": 88, "multipleChoice": 103 }, |
| "art": { "freeform": 109, "multipleChoice": 145 }, |
| "monetization": { "freeform": 116, "multipleChoice": 172 }, |
| "studio": { "freeform": 133, "multipleChoice": 156 }, |
| "learning": { "freeform": 351, "multipleChoice": 376 } |
| } |
| }, |
| "leetcode": { |
| "easy": { |
| "unitTestPassRate": 94.95, |
| "fullPass": 43, |
| "partialPass": 3, |
| "fail": 1, |
| "noCode": 0, |
| "codeQuality": 97.6, |
| "invalidSyntax": 1, |
| "inputTokens": 20240, |
| "outputTokens": 51375 |
| }, |
| "medium": { |
| "unitTestPassRate": 88.3, |
| "fullPass": 76, |
| "partialPass": 11, |
| "fail": 8, |
| "noCode": 2, |
| "codeQuality": 91.6, |
| "invalidSyntax": 3, |
| "inputTokens": 50105, |
| "outputTokens": 656574 |
| }, |
| "hard": { |
| "unitTestPassRate": 67.04, |
| "fullPass": 34, |
| "partialPass": 13, |
| "fail": 9, |
| "noCode": 8, |
| "codeQuality": 68.4, |
| "invalidSyntax": 7, |
| "inputTokens": 31999, |
| "outputTokens": 1117629 |
| } |
| } |
| } |
| } |
| ] |
| } |
|
|
