reports/baseline.json

{
  "model": "Qwen/Qwen3-0.6B",
  "modified_weights": false,
  "device": "cpu",
  "cases": [
    {
      "category": "arithmetic",
      "prompt": "Return only the integer: 17 * 23.",
      "answer": "17 × 23 = 391.  \n\nSo, the integer is **391**.",
      "passed": false,
      "input_tokens": 25,
      "output_tokens": 25,
      "seconds": 5.441395299974829,
      "tokens_per_second": 4.594409819870217,
      "ttft": null,
      "note": "Non-streaming adapter; TTFT not measured"
    },
    {
      "category": "json",
      "prompt": "Return only a JSON object with key \"ready\" and boolean value true.",
      "answer": "```json\n{\n  \"ready\": true\n}\n```",
      "passed": false,
      "input_tokens": 27,
      "output_tokens": 13,
      "seconds": 2.942591800005175,
      "tokens_per_second": 4.417874065977189,
      "ttft": null,
      "note": "Non-streaming adapter; TTFT not measured"
    },
    {
      "category": "ordering",
      "prompt": "Sort these integers ascending. Return only a JSON array: 19, -3, 7, 0.",
      "answer": "```json\n[\n  -3,\n  0,\n  7,\n  19\n]\n```",
      "passed": false,
      "input_tokens": 36,
      "output_tokens": 24,
      "seconds": 5.206944799982011,
      "tokens_per_second": 4.60922881304271,
      "ttft": null,
      "note": "Non-streaming adapter; TTFT not measured"
    },
    {
      "category": "reasoning",
      "prompt": "Every raven is a bird. Some birds swim. Must every raven swim? Answer only yes or no.",
      "answer": "Yes.",
      "passed": false,
      "input_tokens": 35,
      "output_tokens": 3,
      "seconds": 1.0609624999342486,
      "tokens_per_second": 2.8276211460687066,
      "ttft": null,
      "note": "Non-streaming adapter; TTFT not measured"
    }
  ],
  "pass_at_1": 0.0,
  "wilson_95": [
    0,
    0.48990002040399916
  ],
  "latency_seconds": {
    "p50": 2.942591800005175,
    "p95": 5.441395299974829,
    "p99": 5.441395299974829
  },
  "agent": {
    "status": "failed",
    "reason": "invalid_model_output",
    "trace": [
      {
        "state": "OBSERVE",
        "error": "JSONDecodeError"
      },
      {
        "state": "OBSERVE",
        "error": "JSONDecodeError"
      },
      {
        "state": "OBSERVE",
        "error": "JSONDecodeError"
      }
    ]
  },
  "limitations": [
    "Public hand-authored smoke tests; contamination unknown",
    "Four cases cannot rank models",
    "Agent file task is not repository-level coding evaluation",
    "Non-streaming latency only",
    "No private data used"
  ]
}