True2456's picture
Public-ready card: correct numeric/tokenizer conclusion, add accuracy + tool-call validation, REAM rejection, evidence files
31b4aff verified
Raw
History Blame Contribute Delete
3.3 kB
{
"model": "step-3.7-p15-vblend-shared8-head8",
"overall": [
24,
24
],
"by_cat": {
"math": [
14,
14
],
"factual": [
10,
10
]
},
"results": [
{
"cat": "math",
"expected": "330",
"ok": true,
"answer": "330",
"finish": "stop"
},
{
"cat": "math",
"expected": "84",
"ok": true,
"answer": "84",
"finish": "stop"
},
{
"cat": "math",
"expected": "120",
"ok": true,
"answer": "120",
"finish": "stop"
},
{
"cat": "math",
"expected": "355",
"ok": true,
"answer": "355",
"finish": "stop"
},
{
"cat": "math",
"expected": "18",
"ok": true,
"answer": "18",
"finish": "stop"
},
{
"cat": "math",
"expected": "270",
"ok": true,
"answer": "270",
"finish": "stop"
},
{
"cat": "math",
"expected": "101",
"ok": true,
"answer": "101",
"finish": "stop"
},
{
"cat": "math",
"expected": "114",
"ok": true,
"answer": "114",
"finish": "stop"
},
{
"cat": "math",
"expected": "215",
"ok": true,
"answer": "To solve this:\n1. Interpret the formatted times correctly: departure is **09:45",
"finish": "stop"
},
{
"cat": "math",
"expected": "60",
"ok": true,
"answer": "60",
"finish": "stop"
},
{
"cat": "math",
"expected": "1",
"ok": true,
"answer": "1",
"finish": "stop"
},
{
"cat": "math",
"expected": "25",
"ok": true,
"answer": "25",
"finish": "stop"
},
{
"cat": "math",
"expected": "518.4",
"ok": true,
"answer": "518.4",
"finish": "stop"
},
{
"cat": "math",
"expected": "36",
"ok": true,
"answer": "36",
"finish": "stop"
},
{
"cat": "factual",
"expected": "Au",
"ok": true,
"answer": "Au",
"finish": "stop"
},
{
"cat": "factual",
"expected": "1969",
"ok": true,
"answer": "1969",
"finish": "stop"
},
{
"cat": "factual",
"expected": "Ottawa",
"ok": true,
"answer": "Ottawa",
"finish": "stop"
},
{
"cat": "factual",
"expected": "6",
"ok": true,
"answer": "6",
"finish": "stop"
},
{
"cat": "factual",
"expected": "Mars",
"ok": true,
"answer": "Mars",
"finish": "stop"
},
{
"cat": "factual",
"expected": "Pacific",
"ok": true,
"answer": "Pacific",
"finish": "stop"
},
{
"cat": "factual",
"expected": "Shakespeare",
"ok": true,
"answer": "Shakespeare",
"finish": "stop"
},
{
"cat": "factual",
"expected": "0",
"ok": true,
"answer": "0",
"finish": "stop"
},
{
"cat": "factual",
"expected": "carbon dioxide",
"ok": true,
"answer": "Carbon dioxide.",
"finish": "stop"
},
{
"cat": "factual",
"expected": "90",
"ok": true,
"answer": "90",
"finish": "stop"
}
]
}