File size: 5,198 Bytes
28788d0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | {
"meta": {
"updated": "2026-07-15",
"note": "Frontier scores are vendor-published or leaderboard-reported numbers for comparison only. Meltdown is a 3B local agent model — compare on the right axes (agent/tool-calling, latency, cost, privacy), not raw MMLU vs 200B+ models.",
"score_format": "percentage (0-100) unless noted"
},
"benchmarks": {
"humaneval": {
"label": "HumanEval (pass@1)",
"category": "code",
"models": {
"meltdown_q8": { "score": null, "params": "3B", "source": "run benchmarks/run_standard.ps1" },
"qwen2.5_coder_3b_instruct": { "score": 70.1, "params": "3B", "source": "Qwen2.5-Coder technical report (Nov 2024)" },
"gpt_4_1": { "score": 92.0, "params": "~", "source": "OpenAI GPT-4.1 announcement (Apr 2025)" },
"claude_sonnet_4": { "score": 90.2, "params": "~", "source": "Anthropic Claude Sonnet 4 model card" },
"gemini_2_5_pro": { "score": 89.0, "params": "~", "source": "Google Gemini 2.5 Pro benchmarks" }
}
},
"mbpp": {
"label": "MBPP (pass@1)",
"category": "code",
"models": {
"meltdown_q8": { "score": null, "params": "3B", "source": "run benchmarks/run_standard.ps1" },
"qwen2.5_coder_3b_instruct": { "score": 68.4, "params": "3B", "source": "Qwen2.5-Coder technical report" },
"gpt_4_1": { "score": 88.5, "params": "~", "source": "OpenAI GPT-4.1 announcement" },
"claude_sonnet_4": { "score": 87.0, "params": "~", "source": "Anthropic Claude Sonnet 4 model card" },
"gemini_2_5_pro": { "score": 86.5, "params": "~", "source": "Google Gemini 2.5 Pro benchmarks" }
}
},
"gsm8k": {
"label": "GSM8K (8-shot)",
"category": "math",
"models": {
"meltdown_q8": { "score": null, "params": "3B", "source": "run benchmarks/run_standard.ps1" },
"qwen2.5_coder_3b_instruct": { "score": 67.5, "params": "3B", "source": "Qwen2.5-Coder technical report" },
"gpt_4_1": { "score": 95.2, "params": "~", "source": "OpenAI GPT-4.1 announcement" },
"claude_sonnet_4": { "score": 94.0, "params": "~", "source": "Anthropic Claude Sonnet 4 model card" },
"gemini_2_5_pro": { "score": 93.5, "params": "~", "source": "Google Gemini 2.5 Pro benchmarks" }
}
},
"mmlu": {
"label": "MMLU (5-shot)",
"category": "knowledge",
"models": {
"meltdown_q8": { "score": null, "params": "3B", "source": "run benchmarks/run_standard.ps1" },
"qwen2.5_coder_3b_instruct": { "score": 65.9, "params": "3B", "source": "Qwen2.5-Coder technical report" },
"gpt_4_1": { "score": 90.2, "params": "~", "source": "OpenAI GPT-4.1 announcement" },
"claude_sonnet_4": { "score": 88.5, "params": "~", "source": "Anthropic Claude Sonnet 4 model card" },
"gemini_2_5_pro": { "score": 89.0, "params": "~", "source": "Google Gemini 2.5 Pro benchmarks" }
}
},
"ifeval": {
"label": "IFEval (strict)",
"category": "instruction",
"models": {
"meltdown_q8": { "score": null, "params": "3B", "source": "run benchmarks/run_standard.ps1" },
"qwen2.5_coder_3b_instruct": { "score": 55.0, "params": "3B", "source": "Qwen2.5-Coder technical report (approx)" },
"gpt_4_1": { "score": 87.5, "params": "~", "source": "OpenAI GPT-4.1 announcement" },
"claude_sonnet_4": { "score": 86.0, "params": "~", "source": "Anthropic Claude Sonnet 4 model card" },
"gemini_2_5_pro": { "score": 85.5, "params": "~", "source": "Google Gemini 2.5 Pro benchmarks" }
}
},
"bfcl_v3_overall": {
"label": "BFCL v3 Overall (tool calling)",
"category": "agent",
"models": {
"meltdown_q8": { "score": null, "params": "3B", "source": "run benchmarks/run_bfcl.ps1" },
"qwen2.5_coder_3b_instruct": { "score": 72.0, "params": "3B", "source": "BFCL leaderboard (approx)" },
"gpt_4_1": { "score": 88.0, "params": "~", "source": "BFCL leaderboard" },
"claude_sonnet_4": { "score": 87.5, "params": "~", "source": "BFCL leaderboard" },
"gemini_2_5_pro": { "score": 86.0, "params": "~", "source": "BFCL leaderboard" }
}
},
"ape_compliance": {
"label": "APE Protocol Compliance",
"category": "agent",
"models": {
"meltdown_q8": { "score": 68.0, "params": "3B", "source": "benchmarks/run_agent_eval.py (harness-aware, n=50 stratified)" },
"qwen2.5_coder_3b_instruct": { "score": null, "params": "3B", "source": "N/A — not APE-trained" },
"gpt_4_1": { "score": null, "params": "~", "source": "N/A — not APE-trained" },
"claude_sonnet_4": { "score": null, "params": "~", "source": "N/A — not APE-trained" },
"gemini_2_5_pro": { "score": null, "params": "~", "source": "N/A — not APE-trained" }
}
}
},
"display_names": {
"meltdown_q8": "Meltdown Q8 (3B)",
"qwen2.5_coder_3b_instruct": "Qwen2.5-Coder-3B (base)",
"gpt_4_1": "GPT-4.1",
"claude_sonnet_4": "Claude Sonnet 4",
"gemini_2_5_pro": "Gemini 2.5 Pro"
},
"comparison_models": [
"meltdown_q8",
"gpt_4_1",
"claude_sonnet_4",
"gemini_2_5_pro"
]
} |