benchmark / benchmark_checkpoint.json
kishl's picture
Upload benchmark_checkpoint.json with huggingface_hub
4d7ac9f verified
Raw
History Blame Contribute Delete
19.5 kB
{
"models": [
{
"model": "bartowski/SmolLM2-135M-Instruct-GGUF (0.1B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "bartowski/SmolLM2-135M-Instruct-GGUF",
"params": 135000000,
"overall_pass": 12,
"ft_pass": 31,
"temp_pass": 41,
"td_pass": 41,
"sp_pass": 51,
"overall_pct": 13.3,
"ft_pct": 34.4,
"temp_pct": 45.6,
"td_pct": 45.6,
"sp_pct": 56.7,
"categories": {
"filetype": 12.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 60.0
}
},
{
"model": "kishl/smollm2-135m-monkesearch-gguf (0.1B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "kishl/smollm2-135m-monkesearch-gguf",
"params": 135000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "tensorblock/tiny_starcoder_py-GGUF (0.2B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "tensorblock/tiny_starcoder_py-GGUF",
"params": 164000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "tensorblock/smalllocal-GGUF (0.2B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "tensorblock/smalllocal-GGUF",
"params": 200000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "unsloth/gemma-3-270m-it-GGUF (0.3B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "unsloth/gemma-3-270m-it-GGUF",
"params": 270000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "unsloth/functiongemma-270m-it-GGUF (0.3B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "unsloth/functiongemma-270m-it-GGUF",
"params": 270000000,
"overall_pass": 10,
"ft_pass": 29,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 32.2,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "LiquidAI/LFM2.5-350M-GGUF (0.3B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "LiquidAI/LFM2.5-350M-GGUF",
"params": 350000000,
"overall_pass": 18,
"ft_pass": 44,
"temp_pass": 37,
"td_pass": 52,
"sp_pass": 62,
"overall_pct": 20.0,
"ft_pct": 48.9,
"temp_pct": 41.1,
"td_pct": 57.8,
"sp_pct": 68.9,
"categories": {
"filetype": 40.0,
"temporal": 20.0,
"combined": 6.7,
"misc": 53.3
}
},
{
"model": "ibm-granite/granite-4.0-h-350m-GGUF (0.3B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "ibm-granite/granite-4.0-h-350m-GGUF",
"params": 350000000,
"overall_pass": 3,
"ft_pass": 47,
"temp_pass": 31,
"td_pass": 33,
"sp_pass": 58,
"overall_pct": 3.3,
"ft_pct": 52.2,
"temp_pct": 34.4,
"td_pct": 36.7,
"sp_pct": 64.4,
"categories": {
"filetype": 12.0,
"temporal": 5.0,
"combined": 0.0,
"misc": 13.3
}
},
{
"model": "HuggingFaceTB/SmolLM2-360M-Instruct-GGUF (0.4B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "HuggingFaceTB/SmolLM2-360M-Instruct-GGUF",
"params": 360000000,
"overall_pass": 9,
"ft_pass": 31,
"temp_pass": 37,
"td_pass": 43,
"sp_pass": 47,
"overall_pct": 10.0,
"ft_pct": 34.4,
"temp_pct": 41.1,
"td_pct": 47.8,
"sp_pct": 52.2,
"categories": {
"filetype": 4.0,
"temporal": 0.0,
"combined": 3.3,
"misc": 53.3
}
},
{
"model": "Qwen/Qwen2.5-0.5B-Instruct-GGUF (0.5B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "Qwen/Qwen2.5-0.5B-Instruct-GGUF",
"params": 500000000,
"overall_pass": 11,
"ft_pass": 22,
"temp_pass": 45,
"td_pass": 45,
"sp_pass": 51,
"overall_pct": 12.2,
"ft_pct": 24.4,
"temp_pct": 50.0,
"td_pct": 50.0,
"sp_pct": 56.7,
"categories": {
"filetype": 4.0,
"temporal": 0.0,
"combined": 3.3,
"misc": 60.0
}
},
{
"model": "Qwen/Qwen1.5-0.5B-Chat-GGUF (0.5B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "Qwen/Qwen1.5-0.5B-Chat-GGUF",
"params": 500000000,
"overall_pass": 7,
"ft_pass": 25,
"temp_pass": 30,
"td_pass": 38,
"sp_pass": 53,
"overall_pct": 7.8,
"ft_pct": 27.8,
"temp_pct": 33.3,
"td_pct": 42.2,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 46.7
}
},
{
"model": "Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF (0.5B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF",
"params": 500000000,
"overall_pass": 25,
"ft_pass": 36,
"temp_pass": 72,
"td_pass": 62,
"sp_pass": 67,
"overall_pct": 27.8,
"ft_pct": 40.0,
"temp_pct": 80.0,
"td_pct": 68.9,
"sp_pct": 74.4,
"categories": {
"filetype": 48.0,
"temporal": 40.0,
"combined": 23.3,
"misc": 33.3
}
},
{
"model": "ruv/ruvltra-claude-code (0.5B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "ruv/ruvltra-claude-code",
"params": 500000000,
"overall_pass": 21,
"ft_pass": 43,
"temp_pass": 51,
"td_pass": 54,
"sp_pass": 58,
"overall_pct": 23.3,
"ft_pct": 47.8,
"temp_pct": 56.7,
"td_pct": 60.0,
"sp_pct": 64.4,
"categories": {
"filetype": 60.0,
"temporal": 20.0,
"combined": 20.0,
"misc": 33.3
}
},
{
"model": "kishl/qwen-coder-0.5b-monkesearch-gguf (0.5B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "kishl/qwen-coder-0.5b-monkesearch-gguf",
"params": 500000000,
"overall_pass": 61,
"ft_pass": 87,
"temp_pass": 87,
"td_pass": 64,
"sp_pass": 90,
"overall_pct": 67.8,
"ft_pct": 96.7,
"temp_pct": 96.7,
"td_pct": 71.1,
"sp_pct": 100.0,
"categories": {
"filetype": 96.0,
"temporal": 50.0,
"combined": 40.0,
"misc": 100.0
}
},
{
"model": "QuantFactory/NuExtract-1.5-tiny-GGUF (0.5B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "QuantFactory/NuExtract-1.5-tiny-GGUF",
"params": 500000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "tensorblock/bloomz-560m-GGUF (0.6B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "tensorblock/bloomz-560m-GGUF",
"params": 560000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "Qwen/Qwen3-0.6B-GGUF (0.6B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "Qwen/Qwen3-0.6B-GGUF",
"params": 600000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "unsloth/Qwen3.5-0.8B-GGUF (0.8B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "unsloth/Qwen3.5-0.8B-GGUF",
"params": 800000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "Andycurrent/Gemma-3-1B-it-GLM-4.7-Flash-Heretic-Uncensored-Thinking_GGUF (1.0B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "Andycurrent/Gemma-3-1B-it-GLM-4.7-Flash-Heretic-Uncensored-Thinking_GGUF",
"params": 1000000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "openbmb/MiniCPM5-1B-GGUF (1.0B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "openbmb/MiniCPM5-1B-GGUF",
"params": 1000000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "itlwas/phi-1_5-Q4_K_M-GGUF (1.0B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "itlwas/phi-1_5-Q4_K_M-GGUF",
"params": 1000000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "bartowski/Llama-3.2-1B-Instruct-GGUF (1.0B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "bartowski/Llama-3.2-1B-Instruct-GGUF",
"params": 1000000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "unsloth/gemma-3-1b-it-GGUF (1.0B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "unsloth/gemma-3-1b-it-GGUF",
"params": 1000000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "tanujrai/StarCoderBase-1B-GGUF (1.0B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "tanujrai/StarCoderBase-1B-GGUF",
"params": 1000000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "ibm-granite/granite-4.0-h-1b-GGUF (1.0B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "ibm-granite/granite-4.0-h-1b-GGUF",
"params": 1000000000,
"overall_pass": 34,
"ft_pass": 58,
"temp_pass": 83,
"td_pass": 68,
"sp_pass": 59,
"overall_pct": 37.8,
"ft_pct": 64.4,
"temp_pct": 92.2,
"td_pct": 75.6,
"sp_pct": 65.6,
"categories": {
"filetype": 44.0,
"temporal": 70.0,
"combined": 13.3,
"misc": 86.7
}
},
{
"model": "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF (1.1B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF",
"params": 1100000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "LiquidAI/LFM2.5-1.2B-Instruct-GGUF (1.2B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "LiquidAI/LFM2.5-1.2B-Instruct-GGUF",
"params": 1200000000,
"overall_pass": 17,
"ft_pass": 49,
"temp_pass": 57,
"td_pass": 47,
"sp_pass": 55,
"overall_pct": 18.9,
"ft_pct": 54.4,
"temp_pct": 63.3,
"td_pct": 52.2,
"sp_pct": 61.1,
"categories": {
"filetype": 40.0,
"temporal": 50.0,
"combined": 26.7,
"misc": 6.7
}
},
{
"model": "unsloth/DeepSeek-R1-Distill-Qwen-1.5B-GGUF (1.5B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "unsloth/DeepSeek-R1-Distill-Qwen-1.5B-GGUF",
"params": 1500000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "Qwen/Qwen2-1.5B-Instruct-GGUF (1.5B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "Qwen/Qwen2-1.5B-Instruct-GGUF",
"params": 1500000000,
"overall_pass": 28,
"ft_pass": 61,
"temp_pass": 78,
"td_pass": 46,
"sp_pass": 68,
"overall_pct": 31.1,
"ft_pct": 67.8,
"temp_pct": 86.7,
"td_pct": 51.1,
"sp_pct": 75.6,
"categories": {
"filetype": 72.0,
"temporal": 5.0,
"combined": 6.7,
"misc": 53.3
}
},
{
"model": "Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF (1.5B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF",
"params": 1500000000,
"overall_pass": 11,
"ft_pass": 31,
"temp_pass": 41,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 12.2,
"ft_pct": 34.4,
"temp_pct": 45.6,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 4.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
},
{
"model": "lmstudio-community/OpenCoder-1.5B-Instruct-GGUF (1.5B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "lmstudio-community/OpenCoder-1.5B-Instruct-GGUF",
"params": 1500000000,
"overall_pass": 24,
"ft_pass": 31,
"temp_pass": 75,
"td_pass": 65,
"sp_pass": 44,
"overall_pct": 26.7,
"ft_pct": 34.4,
"temp_pct": 83.3,
"td_pct": 72.2,
"sp_pct": 48.9,
"categories": {
"filetype": 24.0,
"temporal": 40.0,
"combined": 36.7,
"misc": 46.7
}
},
{
"model": "Qwen/Qwen3-1.7B-GGUF (1.7B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "Qwen/Qwen3-1.7B-GGUF",
"params": 1700000000,
"overall_pass": 53,
"ft_pass": 72,
"temp_pass": 84,
"td_pass": 71,
"sp_pass": 84,
"overall_pct": 58.9,
"ft_pct": 80.0,
"temp_pct": 93.3,
"td_pct": 78.9,
"sp_pct": 93.3,
"categories": {
"filetype": 68.0,
"temporal": 60.0,
"combined": 43.3,
"misc": 73.3
}
},
{
"model": "HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF (1.7B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF",
"params": 1700000000,
"overall_pass": 27,
"ft_pass": 44,
"temp_pass": 76,
"td_pass": 66,
"sp_pass": 73,
"overall_pct": 30.0,
"ft_pct": 48.9,
"temp_pct": 84.4,
"td_pct": 73.3,
"sp_pct": 81.1,
"categories": {
"filetype": 56.0,
"temporal": 50.0,
"combined": 33.3,
"misc": 26.7
}
},
{
"model": "Qwen/Qwen1.5-1.8B-Chat-GGUF (1.8B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "Qwen/Qwen1.5-1.8B-Chat-GGUF",
"params": 1800000000,
"overall_pass": 9,
"ft_pass": 32,
"temp_pass": 38,
"td_pass": 41,
"sp_pass": 51,
"overall_pct": 10.0,
"ft_pct": 35.6,
"temp_pct": 42.2,
"td_pct": 45.6,
"sp_pct": 56.7,
"categories": {
"filetype": 4.0,
"temporal": 0.0,
"combined": 3.3,
"misc": 53.3
}
},
{
"model": "mradermacher/Qwen3-MOE-4x0.6B-2.4B-Writing-Thunder-V1.2-GGUF (2.4B)",
"total": 90,
"date": "2026-06-19",
"hf_repo": "mradermacher/Qwen3-MOE-4x0.6B-2.4B-Writing-Thunder-V1.2-GGUF",
"params": 2400000000,
"overall_pass": 10,
"ft_pass": 30,
"temp_pass": 40,
"td_pass": 40,
"sp_pass": 53,
"overall_pct": 11.1,
"ft_pct": 33.3,
"temp_pct": 44.4,
"td_pct": 44.4,
"sp_pct": 58.9,
"categories": {
"filetype": 0.0,
"temporal": 0.0,
"combined": 0.0,
"misc": 66.7
}
}
]
}