SenseNova-Loop-9B-Preview-NVFP4 / evaluation /summary_sglang_bf16_mtp.json
gitcommit90's picture
NVFP4 (ModelOpt) quantization of sensenova/SenseNova-Loop-9B-Preview @27abea77 with BF16 MTP head
a57aa9f verified
Raw History Blame Contribute Delete
2.58 kB
{
"eval_version": "sn-loop-eval-v1",
"model": "sensenova-loop-bf16",
"base_url": "http://127.0.0.1:30080/v1",
"started": "2026-10-30T16:28:25-0700",
"enable_thinking": false,
"sampling": {
"temperature": 0.0,
"top_p": 1.0,
"seed": 0,
"max_tokens": {
"ifeval": 1024,
"gsm8k": 1024,
"mmlu_pro": 1024,
"humaneval": 1024,
"tools": 512,
"vqa": 64
}
},
"concurrency": 16,
"limit": null,
"host": "tux",
"data_sha256": {
"ifeval": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
"gsm8k": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
"mmlu_pro": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
"humaneval": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
"tools": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
"vqa": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d"
},
"tasks": {
"gsm8k": {
"n": 80,
"correct": 74,
"accuracy": 0.925,
"wilson95": [
0.8458798383979047,
0.9651747509106547
],
"n_errors": 0,
"n_truncated": 1,
"mean_completion_tokens": 186.425,
"sum_completion_tokens": 14914,
"wall_s": 138.4
},
"humaneval": {
"n": 40,
"correct": 32,
"accuracy": 0.8,
"wilson95": [
0.6524269352437855,
0.8950001032811363
],
"n_errors": 0,
"n_truncated": 0,
"mean_completion_tokens": 230.5,
"sum_completion_tokens": 9220,
"wall_s": 94.4
},
"tools": {
"n": 20,
"correct": 19,
"accuracy": 0.95,
"wilson95": [
0.7638688045357189,
0.9911185512976646
],
"n_errors": 0,
"n_truncated": 0,
"mean_completion_tokens": 64.9,
"sum_completion_tokens": 1298,
"names_correct": 0.95,
"parse_source_counts": {
"server": 16,
"none": 4
},
"format_well_formed": 1.0,
"raw_tool_call_tags_in_content": 0,
"by_category": {
"irrelevance": "3/3",
"multiple": "6/6",
"parallel": "3/3",
"simple": "7/8"
},
"wall_s": 24.8
}
},
"server_models": {
"object": "list",
"data": [
{
"id": "sensenova-loop-bf16",
"object": "model",
"created": 1793402905,
"owned_by": "sglang",
"root": "sensenova-loop-bf16",
"parent": null,
"max_model_len": 32768
}
]
},
"wall_s": 257.7
}