{
  "eval_version": "sn-loop-eval-v1",
  "model": "sensenova-loop-nvfp4",
  "base_url": "http://127.0.0.1:30080/v1",
  "started": "2026-10-30T16:45:48-0700",
  "enable_thinking": false,
  "sampling": {
    "temperature": 0.0,
    "top_p": 1.0,
    "seed": 0,
    "max_tokens": {
      "ifeval": 1024,
      "gsm8k": 1024,
      "mmlu_pro": 1024,
      "humaneval": 1024,
      "tools": 512,
      "vqa": 64
    }
  },
  "concurrency": 16,
  "limit": null,
  "host": "tux",
  "data_sha256": {
    "ifeval": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
    "gsm8k": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
    "mmlu_pro": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
    "humaneval": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
    "tools": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
    "vqa": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d"
  },
  "tasks": {
    "ifeval": {
      "n": 60,
      "correct": 45,
      "accuracy": 0.75,
      "wilson95": [
        0.6276792981955739,
        0.8422347752873125
      ],
      "n_errors": 0,
      "n_truncated": 4,
      "mean_completion_tokens": 272.6666666666667,
      "sum_completion_tokens": 16360,
      "instruction_level_acc": 0.7831325301204819,
      "n_instructions": 83,
      "wall_s": 196.0
    },
    "gsm8k": {
      "n": 80,
      "correct": 76,
      "accuracy": 0.95,
      "wilson95": [
        0.8783772371965909,
        0.9803864456007073
      ],
      "n_errors": 0,
      "n_truncated": 1,
      "mean_completion_tokens": 187.2,
      "sum_completion_tokens": 14976,
      "wall_s": 156.1
    },
    "mmlu_pro": {
      "n": 60,
      "correct": 41,
      "accuracy": 0.6833333333333333,
      "wilson95": [
        0.5576623287169538,
        0.7869413251704964
      ],
      "n_errors": 0,
      "n_truncated": 9,
      "mean_completion_tokens": 588.55,
      "sum_completion_tokens": 35313,
      "wall_s": 352.6
    },
    "humaneval": {
      "n": 40,
      "correct": 32,
      "accuracy": 0.8,
      "wilson95": [
        0.6524269352437855,
        0.8950001032811363
      ],
      "n_errors": 0,
      "n_truncated": 0,
      "mean_completion_tokens": 197.975,
      "sum_completion_tokens": 7919,
      "wall_s": 79.5
    },
    "tools": {
      "n": 20,
      "correct": 19,
      "accuracy": 0.95,
      "wilson95": [
        0.7638688045357189,
        0.9911185512976646
      ],
      "n_errors": 0,
      "n_truncated": 0,
      "mean_completion_tokens": 54.3,
      "sum_completion_tokens": 1086,
      "names_correct": 0.95,
      "parse_source_counts": {
        "server": 16,
        "none": 4
      },
      "format_well_formed": 1.0,
      "raw_tool_call_tags_in_content": 0,
      "by_category": {
        "irrelevance": "3/3",
        "multiple": "5/6",
        "parallel": "3/3",
        "simple": "8/8"
      },
      "wall_s": 24.6
    },
    "vqa": {
      "n": 40,
      "correct": 36,
      "accuracy": 0.9,
      "wilson95": [
        0.7694822464406407,
        0.9604204715925885
      ],
      "n_errors": 0,
      "n_truncated": 0,
      "mean_completion_tokens": 4.075,
      "sum_completion_tokens": 163,
      "vqa_soft_acc": 0.8833333333333334,
      "wall_s": 25.5
    }
  },
  "server_models": {
    "object": "list",
    "data": [
      {
        "id": "sensenova-loop-nvfp4",
        "object": "model",
        "created": 1793403948,
        "owned_by": "sglang",
        "root": "sensenova-loop-nvfp4",
        "parent": null,
        "max_model_len": 32768
      }
    ]
  },
  "wall_s": 834.3
}