Upload folder using huggingface_hub
Browse files- README.md +3 -1
- models.json +16 -0
- plots/perf_per_byte.png +2 -2
- plots/tokens_per_task_vs_active.png +0 -0
- plots/wall_vs_active.png +0 -0
- plots/wall_vs_tokens.png +0 -0
- plots/wall_vs_total.png +0 -0
- results.json +53 -0
- results_full42.csv +1 -0
README.md
CHANGED
|
@@ -109,6 +109,7 @@ those as 39/42, 32/42, or 35/42 would count harness limits as model failures.
|
|
| 109 |
| [Laguna-XS-2.1](https://huggingface.co/Myric/Laguna-XS-2.1-APEX-GGUF) | APEX dynamic v2 | 25/25 | 5/5 | 8/9 | 3/3 | **41/42** | 0 | 2 (41) |
|
| 110 |
| [Laguna-S-2.1](https://huggingface.co/Myric/Laguna-S-2.1-APEX-GGUF) | APEX i-quality v2 | 25/25 | 5/5 | 7/7 | 3/3 | **40/40** | 2 | 1 |
|
| 111 |
| [Nemotron-3-Super-120B-A12B](https://huggingface.co/Myric/Nemotron-3-Super-120B-A12B-APEX-GGUF) | APEX i-quality | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 1 |
|
|
|
|
| 112 |
| [Ornith-1.5-122B](https://huggingface.co/ornith-ai/Ornith-1.5-122B) | APEX i-quality | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 2 (42) |
|
| 113 |
| [Ornith-1.5-122B](https://huggingface.co/ornith-ai/Ornith-1.5-122B) | APEX i-compact | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 2 (42) |
|
| 114 |
| [Ternary-Bonsai-2-27B](https://huggingface.co/prism-ml/Ternary-Bonsai-2-27B-gguf) | PQ2_0 (2.13 bpw) | 25/25 | 5/5 | 8/8 | 3/3 | **41/41** | 1 | 1 |
|
|
@@ -145,7 +146,8 @@ IQ3_XXS), the link goes to the base/official card and the Quant column names the
|
|
| 145 |
With voids excluded, the picture is cleaner — and different from the raw denominators:
|
| 146 |
|
| 147 |
- **Clean 42/42, no voids:** Muse-Glimmer-30B, K2-Horizon v2, Kolibri-1 (APEX and uniform),
|
| 148 |
-
Laguna-XS i-quality v2, Nemotron-3-Super-120B,
|
|
|
|
| 149 |
Their comparison is tokens-at-equal-score, not correctness.
|
| 150 |
- **Perfect on every valid task, via voids:** Spark-X2.6-4B (39/39, 3 void), Spark-X2.5 at
|
| 151 |
BF16 and Q6_K (32/32 and 30/30; 10 and 12 void), MiniCPM5-2B (35/35, 7 void), LFM2.5 Q8_0
|
|
|
|
| 109 |
| [Laguna-XS-2.1](https://huggingface.co/Myric/Laguna-XS-2.1-APEX-GGUF) | APEX dynamic v2 | 25/25 | 5/5 | 8/9 | 3/3 | **41/42** | 0 | 2 (41) |
|
| 110 |
| [Laguna-S-2.1](https://huggingface.co/Myric/Laguna-S-2.1-APEX-GGUF) | APEX i-quality v2 | 25/25 | 5/5 | 7/7 | 3/3 | **40/40** | 2 | 1 |
|
| 111 |
| [Nemotron-3-Super-120B-A12B](https://huggingface.co/Myric/Nemotron-3-Super-120B-A12B-APEX-GGUF) | APEX i-quality | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 1 |
|
| 112 |
+
| [Nemotron-3.5-Super-VL-120B-A12B](https://huggingface.co/Myric/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-APEX-GGUF) | APEX i-quality | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 1 |
|
| 113 |
| [Ornith-1.5-122B](https://huggingface.co/ornith-ai/Ornith-1.5-122B) | APEX i-quality | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 2 (42) |
|
| 114 |
| [Ornith-1.5-122B](https://huggingface.co/ornith-ai/Ornith-1.5-122B) | APEX i-compact | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 2 (42) |
|
| 115 |
| [Ternary-Bonsai-2-27B](https://huggingface.co/prism-ml/Ternary-Bonsai-2-27B-gguf) | PQ2_0 (2.13 bpw) | 25/25 | 5/5 | 8/8 | 3/3 | **41/41** | 1 | 1 |
|
|
|
|
| 146 |
With voids excluded, the picture is cleaner — and different from the raw denominators:
|
| 147 |
|
| 148 |
- **Clean 42/42, no voids:** Muse-Glimmer-30B, K2-Horizon v2, Kolibri-1 (APEX and uniform),
|
| 149 |
+
Laguna-XS i-quality v2, Nemotron-3-Super-120B, Nemotron-3.5-Super-VL-120B-A12B,
|
| 150 |
+
Ornith-1.5-122B (i-quality and i-compact).
|
| 151 |
Their comparison is tokens-at-equal-score, not correctness.
|
| 152 |
- **Perfect on every valid task, via voids:** Spark-X2.6-4B (39/39, 3 void), Spark-X2.5 at
|
| 153 |
BF16 and Q6_K (32/32 and 30/30; 10 and 12 void), MiniCPM5-2B (35/35, 7 void), LFM2.5 Q8_0
|
models.json
CHANGED
|
@@ -126,6 +126,22 @@
|
|
| 126 |
-10
|
| 127 |
]
|
| 128 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
{
|
| 130 |
"group": "ornith122-iquality",
|
| 131 |
"model": "Ornith-1.5-122B",
|
|
|
|
| 126 |
-10
|
| 127 |
]
|
| 128 |
},
|
| 129 |
+
{
|
| 130 |
+
"group": "nemotron35-super-vl-iquality-oc2",
|
| 131 |
+
"model": "Nemotron-3.5-Super-VL-120B-A12B",
|
| 132 |
+
"quant": "APEX i-quality",
|
| 133 |
+
"active_b": 12.0,
|
| 134 |
+
"total_b": 120.0,
|
| 135 |
+
"link": "https://huggingface.co/Myric/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-APEX-GGUF",
|
| 136 |
+
"box": "gb10",
|
| 137 |
+
"annotate": true,
|
| 138 |
+
"short": "Nemotron-3.5-SV",
|
| 139 |
+
"size_gb": 81.03,
|
| 140 |
+
"offset": [
|
| 141 |
+
10,
|
| 142 |
+
8
|
| 143 |
+
]
|
| 144 |
+
},
|
| 145 |
{
|
| 146 |
"group": "ornith122-iquality",
|
| 147 |
"model": "Ornith-1.5-122B",
|
plots/perf_per_byte.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
plots/tokens_per_task_vs_active.png
CHANGED
|
|
plots/wall_vs_active.png
CHANGED
|
|
plots/wall_vs_tokens.png
CHANGED
|
|
plots/wall_vs_total.png
CHANGED
|
|
results.json
CHANGED
|
@@ -1345,6 +1345,59 @@
|
|
| 1345 |
"total_wall_s": 9853.6,
|
| 1346 |
"total_tokens_out": 164349
|
| 1347 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1348 |
{
|
| 1349 |
"label": "nemotron3super-iquality-oc2-r1",
|
| 1350 |
"group": "nemotron3super-iquality-oc2",
|
|
|
|
| 1345 |
"total_wall_s": 9853.6,
|
| 1346 |
"total_tokens_out": 164349
|
| 1347 |
},
|
| 1348 |
+
{
|
| 1349 |
+
"label": "nemotron35-super-vl-iquality-oc2-r1",
|
| 1350 |
+
"group": "nemotron35-super-vl-iquality-oc2",
|
| 1351 |
+
"box": "gb10",
|
| 1352 |
+
"model_path": "NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-APEX-i-quality.gguf",
|
| 1353 |
+
"suites": {
|
| 1354 |
+
"frontier": {
|
| 1355 |
+
"tasks": 9,
|
| 1356 |
+
"valid": 9,
|
| 1357 |
+
"passed": 9,
|
| 1358 |
+
"void": 0,
|
| 1359 |
+
"partial": 1.0,
|
| 1360 |
+
"wall_s": 4253.3,
|
| 1361 |
+
"tokens_out": 55742,
|
| 1362 |
+
"source": "results_nemotron35-super-vl-iquality-oc2-r1_opencode_tasks_frontier.json"
|
| 1363 |
+
},
|
| 1364 |
+
"general": {
|
| 1365 |
+
"tasks": 25,
|
| 1366 |
+
"valid": 25,
|
| 1367 |
+
"passed": 25,
|
| 1368 |
+
"void": 0,
|
| 1369 |
+
"partial": 1.0,
|
| 1370 |
+
"wall_s": 4568.3,
|
| 1371 |
+
"tokens_out": 41086,
|
| 1372 |
+
"source": "results_nemotron35-super-vl-iquality-oc2-r1_opencode_tasks.json"
|
| 1373 |
+
},
|
| 1374 |
+
"hard": {
|
| 1375 |
+
"tasks": 5,
|
| 1376 |
+
"valid": 5,
|
| 1377 |
+
"passed": 5,
|
| 1378 |
+
"void": 0,
|
| 1379 |
+
"partial": 1.0,
|
| 1380 |
+
"wall_s": 1699.7,
|
| 1381 |
+
"tokens_out": 18531,
|
| 1382 |
+
"source": "results_nemotron35-super-vl-iquality-oc2-r1_opencode_tasks_hard.json"
|
| 1383 |
+
},
|
| 1384 |
+
"tier4a": {
|
| 1385 |
+
"tasks": 3,
|
| 1386 |
+
"valid": 3,
|
| 1387 |
+
"passed": 3,
|
| 1388 |
+
"void": 0,
|
| 1389 |
+
"partial": 1.0,
|
| 1390 |
+
"wall_s": 531.6,
|
| 1391 |
+
"tokens_out": 4579,
|
| 1392 |
+
"source": "results_nemotron35-super-vl-iquality-oc2-r1_opencode_tasks_tier4a.json"
|
| 1393 |
+
}
|
| 1394 |
+
},
|
| 1395 |
+
"total_passed": 42,
|
| 1396 |
+
"total_valid": 42,
|
| 1397 |
+
"total_void": 0,
|
| 1398 |
+
"total_wall_s": 11052.9,
|
| 1399 |
+
"total_tokens_out": 119938
|
| 1400 |
+
},
|
| 1401 |
{
|
| 1402 |
"label": "nemotron3super-iquality-oc2-r1",
|
| 1403 |
"group": "nemotron3super-iquality-oc2",
|
results_full42.csv
CHANGED
|
@@ -24,6 +24,7 @@ mellum21-12b-a2.5b-thinking-q8_0-v2-r1,mellum21-12b-a2.5b-thinking-q8_0-v2,gb10,
|
|
| 24 |
mellum21-12b-a2.5b-thinking-q8_0-v2-r2,mellum21-12b-a2.5b-thinking-q8_0-v2,gb10,Mellum2.1-12B-A2.5B-Thinking-Q8_0.gguf,25/25,4/4,8/9,3/3,40/41,1,40,1.53,255666,0.9934
|
| 25 |
minicpm5-2b-q8,minicpm5-2b-q8,gb10,MiniCPM5-2B-Q8_0.gguf,23/23,3/3,6/6,3/3,35/35,7,35,1.9,390175,1.0
|
| 26 |
nemotron35-bolt-74b-nvidia24gb-oc2-r1,nemotron35-bolt-74b-nvidia24gb-oc2,gb10,NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf,24/24,5/5,7/7,3/3,39/39,3,39,2.74,164349,1.0
|
|
|
|
| 27 |
nemotron3super-iquality-oc2-r1,nemotron3super-iquality-oc2,gb10,Nemotron-3-Super-120B-A12B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,3.4,165619,1.0
|
| 28 |
ornith122-icompact-r1,ornith122-icompact,gb10,Ornith-1.5-122B-APEX-i-compact.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.79,132233,1.0
|
| 29 |
ornith122-icompact-r2,ornith122-icompact,gb10,Ornith-1.5-122B-APEX-i-compact.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.83,146077,1.0
|
|
|
|
| 24 |
mellum21-12b-a2.5b-thinking-q8_0-v2-r2,mellum21-12b-a2.5b-thinking-q8_0-v2,gb10,Mellum2.1-12B-A2.5B-Thinking-Q8_0.gguf,25/25,4/4,8/9,3/3,40/41,1,40,1.53,255666,0.9934
|
| 25 |
minicpm5-2b-q8,minicpm5-2b-q8,gb10,MiniCPM5-2B-Q8_0.gguf,23/23,3/3,6/6,3/3,35/35,7,35,1.9,390175,1.0
|
| 26 |
nemotron35-bolt-74b-nvidia24gb-oc2-r1,nemotron35-bolt-74b-nvidia24gb-oc2,gb10,NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf,24/24,5/5,7/7,3/3,39/39,3,39,2.74,164349,1.0
|
| 27 |
+
nemotron35-super-vl-iquality-oc2-r1,nemotron35-super-vl-iquality-oc2,gb10,NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,3.07,119938,1.0
|
| 28 |
nemotron3super-iquality-oc2-r1,nemotron3super-iquality-oc2,gb10,Nemotron-3-Super-120B-A12B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,3.4,165619,1.0
|
| 29 |
ornith122-icompact-r1,ornith122-icompact,gb10,Ornith-1.5-122B-APEX-i-compact.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.79,132233,1.0
|
| 30 |
ornith122-icompact-r2,ornith122-icompact,gb10,Ornith-1.5-122B-APEX-i-compact.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.83,146077,1.0
|