{ "overall": [ { "key": "astra", "name": "GPT-6 Astra (max)", "provider": "OpenAI", "metrics": [ { "label": "Overall", "value": 0.6193352866666667, "currency": false }, { "label": "Text-to-Scene", "value": 0.72382, "currency": false }, { "label": "Image-to-Scene", "value": 0.5148505733333334, "currency": false }, { "label": "Mean cost / case", "value": 12.896991822966507, "currency": true } ] }, { "key": "gemini", "name": "Gemini 3.8 Flash (high)", "provider": "Google", "metrics": [ { "label": "Overall", "value": 0.6189750266666667, "currency": false }, { "label": "Text-to-Scene", "value": 0.65689, "currency": false }, { "label": "Image-to-Scene", "value": 0.5810600533333334, "currency": false }, { "label": "Mean cost / case", "value": 1.9235185771258503, "currency": true } ] }, { "key": "fable", "name": "Claude Fable 5.1 (max)", "provider": "Anthropic", "metrics": [ { "label": "Overall", "value": 0.60604898, "currency": false }, { "label": "Text-to-Scene", "value": 0.78775, "currency": false }, { "label": "Image-to-Scene", "value": 0.4243479600000001, "currency": false }, { "label": "Mean cost / case", "value": 10.623258766403161, "currency": true } ] }, { "key": "opus", "name": "Claude Opus 5 (max)", "provider": "Anthropic", "metrics": [ { "label": "Overall", "value": 0.5931299933333334, "currency": false }, { "label": "Text-to-Scene", "value": 0.7181599999999999, "currency": false }, { "label": "Image-to-Scene", "value": 0.46809998666666675, "currency": false }, { "label": "Mean cost / case", "value": 19.47789116793412, "currency": true } ] }, { "key": "sol", "name": "GPT-5.6 Sol (high)", "provider": "OpenAI", "metrics": [ { "label": "Overall", "value": 0.5499994666666667, "currency": false }, { "label": "Text-to-Scene", "value": 0.70704, "currency": false }, { "label": "Image-to-Scene", "value": 0.3929589333333333, "currency": false }, { "label": "Mean cost / case", "value": 2.9749956842592593, "currency": true } ] }, { "key": "muse", "name": "Muse Spark 1.3 (medium)", "provider": "Meta", "metrics": [ { "label": "Overall", "value": 0.5022053666666666, "currency": false }, { "label": "Text-to-Scene", "value": 0.646195, "currency": false }, { "label": "Image-to-Scene", "value": 0.35821573333333334, "currency": false }, { "label": "Mean cost / case", "value": 0.0861066423076923, "currency": true } ] }, { "key": "glm", "name": "GLM-5.3 Flash (max)", "provider": "Z.ai", "metrics": [ { "label": "Overall", "value": 0.4147088666666666, "currency": false }, { "label": "Text-to-Scene", "value": 0.5094299999999999, "currency": false }, { "label": "Image-to-Scene", "value": 0.3199877333333333, "currency": false }, { "label": "Mean cost / case", "value": 0.40130584861111107, "currency": true } ] }, { "key": "qwen-off", "name": "Qwen 3.8 27B (thinking off)", "provider": "Alibaba", "metrics": [ { "label": "Overall", "value": 0.37818169333333335, "currency": false }, { "label": "Text-to-Scene", "value": 0.55666, "currency": false }, { "label": "Image-to-Scene", "value": 0.19970338666666668, "currency": false }, { "label": "Mean cost / case", "value": 0.38413493183103564, "currency": true } ] }, { "key": "grok", "name": "Grok 4.6 (high)", "provider": "xAI", "metrics": [ { "label": "Overall", "value": 0.37567506, "currency": false }, { "label": "Text-to-Scene", "value": 0.56721, "currency": false }, { "label": "Image-to-Scene", "value": 0.18414012, "currency": false }, { "label": "Mean cost / case", "value": 2.2948689296296294, "currency": true } ] }, { "key": "qwen-on", "name": "Qwen 3.8 27B (thinking on)", "provider": "Alibaba", "metrics": [ { "label": "Overall", "value": 0.37094399333333333, "currency": false }, { "label": "Text-to-Scene", "value": 0.51571, "currency": false }, { "label": "Image-to-Scene", "value": 0.2261779866666667, "currency": false }, { "label": "Mean cost / case", "value": 0.29748765575757574, "currency": true } ] }, { "key": "inkling", "name": "Inkling (high)", "provider": "Thinking Machines", "metrics": [ { "label": "Overall", "value": 0.31150237333333336, "currency": false }, { "label": "Text-to-Scene", "value": 0.42498500000000006, "currency": false }, { "label": "Image-to-Scene", "value": 0.19801974666666666, "currency": false }, { "label": "Mean cost / case", "value": 0.4783280191269841, "currency": true } ] }, { "key": "gemma-on", "name": "Gemma 4 31B (thinking on)", "provider": "Google", "metrics": [ { "label": "Overall", "value": 0.29812257999999997, "currency": false }, { "label": "Text-to-Scene", "value": 0.470705, "currency": false }, { "label": "Image-to-Scene", "value": 0.12554015999999998, "currency": false }, { "label": "Mean cost / case", "value": 0.026867125396825398, "currency": true } ] }, { "key": "gemma-off", "name": "Gemma 4 31B (thinking off)", "provider": "Google", "metrics": [ { "label": "Overall", "value": 0.28498897333333334, "currency": false }, { "label": "Text-to-Scene", "value": 0.45494, "currency": false }, { "label": "Image-to-Scene", "value": 0.11503794666666667, "currency": false }, { "label": "Mean cost / case", "value": 0.017148714965986397, "currency": true } ] }, { "key": "deepseek", "name": "DeepSeek V4.1 Flash (high)", "provider": "DeepSeek", "metrics": [ { "label": "Overall", "value": 0.22658958, "currency": false }, { "label": "Text-to-Scene", "value": 0.24269, "currency": false }, { "label": "Image-to-Scene", "value": 0.21048916, "currency": false }, { "label": "Mean cost / case", "value": 0.1240297601010101, "currency": true } ] } ], "t2s": [ { "key": "fable", "name": "Claude Fable 5.1 (max)", "provider": "Anthropic", "metrics": [ { "label": "Text-to-Scene", "value": 0.78775, "currency": false }, { "label": "Detailed Alignment", "value": 0.7253149999999999, "currency": false }, { "label": "Overview Alignment", "value": 0.7717050000000001, "currency": false }, { "label": "Physical Safety", "value": 0.898, "currency": false }, { "label": "Mean cost / case", "value": 18.1576316, "currency": true } ] }, { "key": "astra", "name": "GPT-6 Astra (max)", "provider": "OpenAI", "metrics": [ { "label": "Text-to-Scene", "value": 0.72382, "currency": false }, { "label": "Detailed Alignment", "value": 0.707165, "currency": false }, { "label": "Overview Alignment", "value": 0.733235, "currency": false }, { "label": "Physical Safety", "value": 0.712, "currency": false }, { "label": "Mean cost / case", "value": 22.472260736842106, "currency": true } ] }, { "key": "opus", "name": "Claude Opus 5 (max)", "provider": "Anthropic", "metrics": [ { "label": "Text-to-Scene", "value": 0.7181599999999999, "currency": false }, { "label": "Detailed Alignment", "value": 0.63106, "currency": false }, { "label": "Overview Alignment", "value": 0.71617, "currency": false }, { "label": "Physical Safety", "value": 0.811, "currency": false }, { "label": "Mean cost / case", "value": 34.49611078947368, "currency": true } ] }, { "key": "sol", "name": "GPT-5.6 Sol (high)", "provider": "OpenAI", "metrics": [ { "label": "Text-to-Scene", "value": 0.70704, "currency": false }, { "label": "Detailed Alignment", "value": 0.63186, "currency": false }, { "label": "Overview Alignment", "value": 0.710485, "currency": false }, { "label": "Physical Safety", "value": 0.772, "currency": false }, { "label": "Mean cost / case", "value": 4.07260365, "currency": true } ] }, { "key": "gemini", "name": "Gemini 3.8 Flash (high)", "provider": "Google", "metrics": [ { "label": "Text-to-Scene", "value": 0.65689, "currency": false }, { "label": "Detailed Alignment", "value": 0.6369100000000001, "currency": false }, { "label": "Overview Alignment", "value": 0.66406, "currency": false }, { "label": "Physical Safety", "value": 0.655, "currency": false }, { "label": "Mean cost / case", "value": 2.49471265, "currency": true } ] }, { "key": "muse", "name": "Muse Spark 1.3 (medium)", "provider": "Meta", "metrics": [ { "label": "Text-to-Scene", "value": 0.646195, "currency": false }, { "label": "Detailed Alignment", "value": 0.5981099999999999, "currency": false }, { "label": "Overview Alignment", "value": 0.660675, "currency": false }, { "label": "Physical Safety", "value": 0.651, "currency": false }, { "label": "Mean cost / case", "value": 0.10048539999999999, "currency": true } ] }, { "key": "grok", "name": "Grok 4.6 (high)", "provider": "xAI", "metrics": [ { "label": "Text-to-Scene", "value": 0.56721, "currency": false }, { "label": "Detailed Alignment", "value": 0.52211, "currency": false }, { "label": "Overview Alignment", "value": 0.5764199999999999, "currency": false }, { "label": "Physical Safety", "value": 0.585, "currency": false }, { "label": "Mean cost / case", "value": 4.326270533333333, "currency": true } ] }, { "key": "qwen-off", "name": "Qwen 3.8 27B (thinking off)", "provider": "Alibaba", "metrics": [ { "label": "Text-to-Scene", "value": 0.55666, "currency": false }, { "label": "Detailed Alignment", "value": 0.506755, "currency": false }, { "label": "Overview Alignment", "value": 0.529525, "currency": false }, { "label": "Physical Safety", "value": 0.688, "currency": false }, { "label": "Mean cost / case", "value": 0.49154169999999997, "currency": true } ] }, { "key": "qwen-on", "name": "Qwen 3.8 27B (thinking on)", "provider": "Alibaba", "metrics": [ { "label": "Text-to-Scene", "value": 0.51571, "currency": false }, { "label": "Detailed Alignment", "value": 0.50522, "currency": false }, { "label": "Overview Alignment", "value": 0.45978, "currency": false }, { "label": "Physical Safety", "value": 0.694, "currency": false }, { "label": "Mean cost / case", "value": 0.3344125, "currency": true } ] }, { "key": "glm", "name": "GLM-5.3 Flash (max)", "provider": "Z.ai", "metrics": [ { "label": "Text-to-Scene", "value": 0.5094299999999999, "currency": false }, { "label": "Detailed Alignment", "value": 0.403365, "currency": false }, { "label": "Overview Alignment", "value": 0.537015, "currency": false }, { "label": "Physical Safety", "value": 0.533, "currency": false }, { "label": "Mean cost / case", "value": 0.710521125, "currency": true } ] }, { "key": "gemma-on", "name": "Gemma 4 31B (thinking on)", "provider": "Google", "metrics": [ { "label": "Text-to-Scene", "value": 0.470705, "currency": false }, { "label": "Detailed Alignment", "value": 0.4117, "currency": false }, { "label": "Overview Alignment", "value": 0.41593, "currency": false }, { "label": "Physical Safety", "value": 0.694, "currency": false }, { "label": "Mean cost / case", "value": 0.0253001, "currency": true } ] }, { "key": "gemma-off", "name": "Gemma 4 31B (thinking off)", "provider": "Google", "metrics": [ { "label": "Text-to-Scene", "value": 0.45494, "currency": false }, { "label": "Detailed Alignment", "value": 0.43810000000000004, "currency": false }, { "label": "Overview Alignment", "value": 0.370465, "currency": false }, { "label": "Physical Safety", "value": 0.725, "currency": false }, { "label": "Mean cost / case", "value": 0.0151612, "currency": true } ] }, { "key": "inkling", "name": "Inkling (high)", "provider": "Thinking Machines", "metrics": [ { "label": "Text-to-Scene", "value": 0.42498500000000006, "currency": false }, { "label": "Detailed Alignment", "value": 0.392605, "currency": false }, { "label": "Overview Alignment", "value": 0.336705, "currency": false }, { "label": "Physical Safety", "value": 0.722, "currency": false }, { "label": "Mean cost / case", "value": 0.20252835, "currency": true } ] }, { "key": "deepseek", "name": "DeepSeek V4.1 Flash (high)", "provider": "DeepSeek", "metrics": [ { "label": "Text-to-Scene", "value": 0.24269, "currency": false }, { "label": "Detailed Alignment", "value": 0.216595, "currency": false }, { "label": "Overview Alignment", "value": 0.16134500000000002, "currency": false }, { "label": "Physical Safety", "value": 0.513, "currency": false }, { "label": "Mean cost / case", "value": 0.193865, "currency": true } ] } ], "i2s": [ { "key": "gemini", "name": "Gemini 3.8 Flash (high)", "provider": "Google", "metrics": [ { "label": "Image-to-Scene", "value": 0.5810600533333334, "currency": false }, { "label": "Repair F1", "value": 0.527, "currency": false }, { "label": "Physical Safety", "value": 0.796, "currency": false }, { "label": "Indoor", "value": 0.71653332, "currency": false }, { "label": "Outdoor", "value": 0.5133234200000001, "currency": false }, { "label": "Mean cost / case", "value": 1.3523245042517007, "currency": true } ] }, { "key": "astra", "name": "GPT-6 Astra (max)", "provider": "OpenAI", "metrics": [ { "label": "Image-to-Scene", "value": 0.5148505733333334, "currency": false }, { "label": "Repair F1", "value": 0.445, "currency": false }, { "label": "Physical Safety", "value": 0.796, "currency": false }, { "label": "Indoor", "value": 0.78546656, "currency": false }, { "label": "Outdoor", "value": 0.37954258, "currency": false }, { "label": "Mean cost / case", "value": 3.3217229090909095, "currency": true } ] }, { "key": "opus", "name": "Claude Opus 5 (max)", "provider": "Anthropic", "metrics": [ { "label": "Image-to-Scene", "value": 0.46809998666666675, "currency": false }, { "label": "Repair F1", "value": 0.389, "currency": false }, { "label": "Physical Safety", "value": 0.786, "currency": false }, { "label": "Indoor", "value": 0.61843384, "currency": false }, { "label": "Outdoor", "value": 0.39293306, "currency": false }, { "label": "Mean cost / case", "value": 4.459671546394558, "currency": true } ] }, { "key": "fable", "name": "Claude Fable 5.1 (max)", "provider": "Anthropic", "metrics": [ { "label": "Image-to-Scene", "value": 0.4243479600000001, "currency": false }, { "label": "Repair F1", "value": 0.332, "currency": false }, { "label": "Physical Safety", "value": 0.795, "currency": false }, { "label": "Indoor", "value": 0.53523332, "currency": false }, { "label": "Outdoor", "value": 0.36890528000000006, "currency": false }, { "label": "Mean cost / case", "value": 3.0888859328063245, "currency": true } ] }, { "key": "sol", "name": "GPT-5.6 Sol (high)", "provider": "OpenAI", "metrics": [ { "label": "Image-to-Scene", "value": 0.3929589333333333, "currency": false }, { "label": "Repair F1", "value": 0.319, "currency": false }, { "label": "Physical Safety", "value": 0.689, "currency": false }, { "label": "Indoor", "value": 0.49742972, "currency": false }, { "label": "Outdoor", "value": 0.34072354, "currency": false }, { "label": "Mean cost / case", "value": 1.8773877185185188, "currency": true } ] }, { "key": "muse", "name": "Muse Spark 1.3 (medium)", "provider": "Meta", "metrics": [ { "label": "Image-to-Scene", "value": 0.35821573333333334, "currency": false }, { "label": "Repair F1", "value": 0.246, "currency": false }, { "label": "Physical Safety", "value": 0.807, "currency": false }, { "label": "Indoor", "value": 0.63253332, "currency": false }, { "label": "Outdoor", "value": 0.22105694, "currency": false }, { "label": "Mean cost / case", "value": 0.07172788461538461, "currency": true } ] }, { "key": "glm", "name": "GLM-5.3 Flash (max)", "provider": "Z.ai", "metrics": [ { "label": "Image-to-Scene", "value": 0.3199877333333333, "currency": false }, { "label": "Repair F1", "value": 0.188, "currency": false }, { "label": "Physical Safety", "value": 0.848, "currency": false }, { "label": "Indoor", "value": 0.46533411999999996, "currency": false }, { "label": "Outdoor", "value": 0.24731454, "currency": false }, { "label": "Mean cost / case", "value": 0.09209057222222222, "currency": true } ] }, { "key": "qwen-on", "name": "Qwen 3.8 27B (thinking on)", "provider": "Alibaba", "metrics": [ { "label": "Image-to-Scene", "value": 0.2261779866666667, "currency": false }, { "label": "Repair F1", "value": 0.118, "currency": false }, { "label": "Physical Safety", "value": 0.66, "currency": false }, { "label": "Indoor", "value": 0.29651112, "currency": false }, { "label": "Outdoor", "value": 0.19101142, "currency": false }, { "label": "Mean cost / case", "value": 0.2605628115151515, "currency": true } ] }, { "key": "deepseek", "name": "DeepSeek V4.1 Flash (high)", "provider": "DeepSeek", "metrics": [ { "label": "Image-to-Scene", "value": 0.21048916, "currency": false }, { "label": "Repair F1", "value": 0.066, "currency": false }, { "label": "Physical Safety", "value": 0.787, "currency": false }, { "label": "Indoor", "value": 0.18933412, "currency": false }, { "label": "Outdoor", "value": 0.22106668, "currency": false }, { "label": "Mean cost / case", "value": 0.054194520202020195, "currency": true } ] }, { "key": "qwen-off", "name": "Qwen 3.8 27B (thinking off)", "provider": "Alibaba", "metrics": [ { "label": "Image-to-Scene", "value": 0.19970338666666668, "currency": false }, { "label": "Repair F1", "value": 0.083, "currency": false }, { "label": "Physical Safety", "value": 0.667, "currency": false }, { "label": "Indoor", "value": 0.26629816, "currency": false }, { "label": "Outdoor", "value": 0.166406, "currency": false }, { "label": "Mean cost / case", "value": 0.27672816366207137, "currency": true } ] }, { "key": "inkling", "name": "Inkling (high)", "provider": "Thinking Machines", "metrics": [ { "label": "Image-to-Scene", "value": 0.19801974666666666, "currency": false }, { "label": "Repair F1", "value": 0.095, "currency": false }, { "label": "Physical Safety", "value": 0.609, "currency": false }, { "label": "Indoor", "value": 0.1690288, "currency": false }, { "label": "Outdoor", "value": 0.21251522, "currency": false }, { "label": "Mean cost / case", "value": 0.7541276882539681, "currency": true } ] }, { "key": "grok", "name": "Grok 4.6 (high)", "provider": "xAI", "metrics": [ { "label": "Image-to-Scene", "value": 0.18414012, "currency": false }, { "label": "Repair F1", "value": 0.044, "currency": false }, { "label": "Physical Safety", "value": 0.744, "currency": false }, { "label": "Indoor", "value": 0.1968008, "currency": false }, { "label": "Outdoor", "value": 0.17780978, "currency": false }, { "label": "Mean cost / case", "value": 0.2634673259259259, "currency": true } ] }, { "key": "gemma-on", "name": "Gemma 4 31B (thinking on)", "provider": "Google", "metrics": [ { "label": "Image-to-Scene", "value": 0.12554015999999998, "currency": false }, { "label": "Repair F1", "value": 0.003, "currency": false }, { "label": "Physical Safety", "value": 0.615, "currency": false }, { "label": "Indoor", "value": 0.12704028, "currency": false }, { "label": "Outdoor", "value": 0.1247901, "currency": false }, { "label": "Mean cost / case", "value": 0.028434150793650796, "currency": true } ] }, { "key": "gemma-off", "name": "Gemma 4 31B (thinking off)", "provider": "Google", "metrics": [ { "label": "Image-to-Scene", "value": 0.11503794666666667, "currency": false }, { "label": "Repair F1", "value": 0.001, "currency": false }, { "label": "Physical Safety", "value": 0.572, "currency": false }, { "label": "Indoor", "value": 0.11239600000000001, "currency": false }, { "label": "Outdoor", "value": 0.11635892, "currency": false }, { "label": "Mean cost / case", "value": 0.01913622993197279, "currency": true } ] } ] }