File size: 2,625 Bytes
12496fc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 | {
"model": "Qwen/Qwen3-0.6B",
"modified_weights": false,
"device": "cpu",
"cases": [
{
"category": "arithmetic",
"prompt": "Return only the integer: 17 * 23.",
"answer": "17 × 23 = 391. \n\nSo, the integer is **391**.",
"passed": false,
"input_tokens": 25,
"output_tokens": 25,
"seconds": 5.441395299974829,
"tokens_per_second": 4.594409819870217,
"ttft": null,
"note": "Non-streaming adapter; TTFT not measured"
},
{
"category": "json",
"prompt": "Return only a JSON object with key \"ready\" and boolean value true.",
"answer": "```json\n{\n \"ready\": true\n}\n```",
"passed": false,
"input_tokens": 27,
"output_tokens": 13,
"seconds": 2.942591800005175,
"tokens_per_second": 4.417874065977189,
"ttft": null,
"note": "Non-streaming adapter; TTFT not measured"
},
{
"category": "ordering",
"prompt": "Sort these integers ascending. Return only a JSON array: 19, -3, 7, 0.",
"answer": "```json\n[\n -3,\n 0,\n 7,\n 19\n]\n```",
"passed": false,
"input_tokens": 36,
"output_tokens": 24,
"seconds": 5.206944799982011,
"tokens_per_second": 4.60922881304271,
"ttft": null,
"note": "Non-streaming adapter; TTFT not measured"
},
{
"category": "reasoning",
"prompt": "Every raven is a bird. Some birds swim. Must every raven swim? Answer only yes or no.",
"answer": "Yes.",
"passed": false,
"input_tokens": 35,
"output_tokens": 3,
"seconds": 1.0609624999342486,
"tokens_per_second": 2.8276211460687066,
"ttft": null,
"note": "Non-streaming adapter; TTFT not measured"
}
],
"pass_at_1": 0.0,
"wilson_95": [
0,
0.48990002040399916
],
"latency_seconds": {
"p50": 2.942591800005175,
"p95": 5.441395299974829,
"p99": 5.441395299974829
},
"agent": {
"status": "failed",
"reason": "invalid_model_output",
"trace": [
{
"state": "OBSERVE",
"error": "JSONDecodeError"
},
{
"state": "OBSERVE",
"error": "JSONDecodeError"
},
{
"state": "OBSERVE",
"error": "JSONDecodeError"
}
]
},
"limitations": [
"Public hand-authored smoke tests; contamination unknown",
"Four cases cannot rank models",
"Agent file task is not repository-level coding evaluation",
"Non-streaming latency only",
"No private data used"
]
} |