Myric commited on
Commit
4e1f2a9
·
verified ·
1 Parent(s): 5762307

Upload folder using huggingface_hub

Browse files
README.md CHANGED
@@ -109,6 +109,7 @@ those as 39/42, 32/42, or 35/42 would count harness limits as model failures.
109
  | [Laguna-XS-2.1](https://huggingface.co/Myric/Laguna-XS-2.1-APEX-GGUF) | APEX dynamic v2 | 25/25 | 5/5 | 8/9 | 3/3 | **41/42** | 0 | 2 (41) |
110
  | [Laguna-S-2.1](https://huggingface.co/Myric/Laguna-S-2.1-APEX-GGUF) | APEX i-quality v2 | 25/25 | 5/5 | 7/7 | 3/3 | **40/40** | 2 | 1 |
111
  | [Nemotron-3-Super-120B-A12B](https://huggingface.co/Myric/Nemotron-3-Super-120B-A12B-APEX-GGUF) | APEX i-quality | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 1 |
 
112
  | [Ornith-1.5-122B](https://huggingface.co/ornith-ai/Ornith-1.5-122B) | APEX i-quality | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 2 (42) |
113
  | [Ornith-1.5-122B](https://huggingface.co/ornith-ai/Ornith-1.5-122B) | APEX i-compact | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 2 (42) |
114
  | [Ternary-Bonsai-2-27B](https://huggingface.co/prism-ml/Ternary-Bonsai-2-27B-gguf) | PQ2_0 (2.13 bpw) | 25/25 | 5/5 | 8/8 | 3/3 | **41/41** | 1 | 1 |
@@ -145,7 +146,8 @@ IQ3_XXS), the link goes to the base/official card and the Quant column names the
145
  With voids excluded, the picture is cleaner — and different from the raw denominators:
146
 
147
  - **Clean 42/42, no voids:** Muse-Glimmer-30B, K2-Horizon v2, Kolibri-1 (APEX and uniform),
148
- Laguna-XS i-quality v2, Nemotron-3-Super-120B, Ornith-1.5-122B (i-quality and i-compact).
 
149
  Their comparison is tokens-at-equal-score, not correctness.
150
  - **Perfect on every valid task, via voids:** Spark-X2.6-4B (39/39, 3 void), Spark-X2.5 at
151
  BF16 and Q6_K (32/32 and 30/30; 10 and 12 void), MiniCPM5-2B (35/35, 7 void), LFM2.5 Q8_0
 
109
  | [Laguna-XS-2.1](https://huggingface.co/Myric/Laguna-XS-2.1-APEX-GGUF) | APEX dynamic v2 | 25/25 | 5/5 | 8/9 | 3/3 | **41/42** | 0 | 2 (41) |
110
  | [Laguna-S-2.1](https://huggingface.co/Myric/Laguna-S-2.1-APEX-GGUF) | APEX i-quality v2 | 25/25 | 5/5 | 7/7 | 3/3 | **40/40** | 2 | 1 |
111
  | [Nemotron-3-Super-120B-A12B](https://huggingface.co/Myric/Nemotron-3-Super-120B-A12B-APEX-GGUF) | APEX i-quality | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 1 |
112
+ | [Nemotron-3.5-Super-VL-120B-A12B](https://huggingface.co/Myric/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-APEX-GGUF) | APEX i-quality | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 1 |
113
  | [Ornith-1.5-122B](https://huggingface.co/ornith-ai/Ornith-1.5-122B) | APEX i-quality | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 2 (42) |
114
  | [Ornith-1.5-122B](https://huggingface.co/ornith-ai/Ornith-1.5-122B) | APEX i-compact | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 2 (42) |
115
  | [Ternary-Bonsai-2-27B](https://huggingface.co/prism-ml/Ternary-Bonsai-2-27B-gguf) | PQ2_0 (2.13 bpw) | 25/25 | 5/5 | 8/8 | 3/3 | **41/41** | 1 | 1 |
 
146
  With voids excluded, the picture is cleaner — and different from the raw denominators:
147
 
148
  - **Clean 42/42, no voids:** Muse-Glimmer-30B, K2-Horizon v2, Kolibri-1 (APEX and uniform),
149
+ Laguna-XS i-quality v2, Nemotron-3-Super-120B, Nemotron-3.5-Super-VL-120B-A12B,
150
+ Ornith-1.5-122B (i-quality and i-compact).
151
  Their comparison is tokens-at-equal-score, not correctness.
152
  - **Perfect on every valid task, via voids:** Spark-X2.6-4B (39/39, 3 void), Spark-X2.5 at
153
  BF16 and Q6_K (32/32 and 30/30; 10 and 12 void), MiniCPM5-2B (35/35, 7 void), LFM2.5 Q8_0
models.json CHANGED
@@ -126,6 +126,22 @@
126
  -10
127
  ]
128
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
129
  {
130
  "group": "ornith122-iquality",
131
  "model": "Ornith-1.5-122B",
 
126
  -10
127
  ]
128
  },
129
+ {
130
+ "group": "nemotron35-super-vl-iquality-oc2",
131
+ "model": "Nemotron-3.5-Super-VL-120B-A12B",
132
+ "quant": "APEX i-quality",
133
+ "active_b": 12.0,
134
+ "total_b": 120.0,
135
+ "link": "https://huggingface.co/Myric/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-APEX-GGUF",
136
+ "box": "gb10",
137
+ "annotate": true,
138
+ "short": "Nemotron-3.5-SV",
139
+ "size_gb": 81.03,
140
+ "offset": [
141
+ 10,
142
+ 8
143
+ ]
144
+ },
145
  {
146
  "group": "ornith122-iquality",
147
  "model": "Ornith-1.5-122B",
plots/perf_per_byte.png CHANGED

Git LFS Details

  • SHA256: 1e48b02232dbe4bdab8444f136583281d5335cbb1324642e6abe5b983c9edbd9
  • Pointer size: 131 Bytes
  • Size of remote file: 126 kB

Git LFS Details

  • SHA256: 06ee092074d644773f3284505bc747592cbd18b96f9c645549d12b6780dc70c4
  • Pointer size: 131 Bytes
  • Size of remote file: 129 kB
plots/tokens_per_task_vs_active.png CHANGED
plots/wall_vs_active.png CHANGED
plots/wall_vs_tokens.png CHANGED
plots/wall_vs_total.png CHANGED
results.json CHANGED
@@ -1345,6 +1345,59 @@
1345
  "total_wall_s": 9853.6,
1346
  "total_tokens_out": 164349
1347
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1348
  {
1349
  "label": "nemotron3super-iquality-oc2-r1",
1350
  "group": "nemotron3super-iquality-oc2",
 
1345
  "total_wall_s": 9853.6,
1346
  "total_tokens_out": 164349
1347
  },
1348
+ {
1349
+ "label": "nemotron35-super-vl-iquality-oc2-r1",
1350
+ "group": "nemotron35-super-vl-iquality-oc2",
1351
+ "box": "gb10",
1352
+ "model_path": "NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-APEX-i-quality.gguf",
1353
+ "suites": {
1354
+ "frontier": {
1355
+ "tasks": 9,
1356
+ "valid": 9,
1357
+ "passed": 9,
1358
+ "void": 0,
1359
+ "partial": 1.0,
1360
+ "wall_s": 4253.3,
1361
+ "tokens_out": 55742,
1362
+ "source": "results_nemotron35-super-vl-iquality-oc2-r1_opencode_tasks_frontier.json"
1363
+ },
1364
+ "general": {
1365
+ "tasks": 25,
1366
+ "valid": 25,
1367
+ "passed": 25,
1368
+ "void": 0,
1369
+ "partial": 1.0,
1370
+ "wall_s": 4568.3,
1371
+ "tokens_out": 41086,
1372
+ "source": "results_nemotron35-super-vl-iquality-oc2-r1_opencode_tasks.json"
1373
+ },
1374
+ "hard": {
1375
+ "tasks": 5,
1376
+ "valid": 5,
1377
+ "passed": 5,
1378
+ "void": 0,
1379
+ "partial": 1.0,
1380
+ "wall_s": 1699.7,
1381
+ "tokens_out": 18531,
1382
+ "source": "results_nemotron35-super-vl-iquality-oc2-r1_opencode_tasks_hard.json"
1383
+ },
1384
+ "tier4a": {
1385
+ "tasks": 3,
1386
+ "valid": 3,
1387
+ "passed": 3,
1388
+ "void": 0,
1389
+ "partial": 1.0,
1390
+ "wall_s": 531.6,
1391
+ "tokens_out": 4579,
1392
+ "source": "results_nemotron35-super-vl-iquality-oc2-r1_opencode_tasks_tier4a.json"
1393
+ }
1394
+ },
1395
+ "total_passed": 42,
1396
+ "total_valid": 42,
1397
+ "total_void": 0,
1398
+ "total_wall_s": 11052.9,
1399
+ "total_tokens_out": 119938
1400
+ },
1401
  {
1402
  "label": "nemotron3super-iquality-oc2-r1",
1403
  "group": "nemotron3super-iquality-oc2",
results_full42.csv CHANGED
@@ -24,6 +24,7 @@ mellum21-12b-a2.5b-thinking-q8_0-v2-r1,mellum21-12b-a2.5b-thinking-q8_0-v2,gb10,
24
  mellum21-12b-a2.5b-thinking-q8_0-v2-r2,mellum21-12b-a2.5b-thinking-q8_0-v2,gb10,Mellum2.1-12B-A2.5B-Thinking-Q8_0.gguf,25/25,4/4,8/9,3/3,40/41,1,40,1.53,255666,0.9934
25
  minicpm5-2b-q8,minicpm5-2b-q8,gb10,MiniCPM5-2B-Q8_0.gguf,23/23,3/3,6/6,3/3,35/35,7,35,1.9,390175,1.0
26
  nemotron35-bolt-74b-nvidia24gb-oc2-r1,nemotron35-bolt-74b-nvidia24gb-oc2,gb10,NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf,24/24,5/5,7/7,3/3,39/39,3,39,2.74,164349,1.0
 
27
  nemotron3super-iquality-oc2-r1,nemotron3super-iquality-oc2,gb10,Nemotron-3-Super-120B-A12B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,3.4,165619,1.0
28
  ornith122-icompact-r1,ornith122-icompact,gb10,Ornith-1.5-122B-APEX-i-compact.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.79,132233,1.0
29
  ornith122-icompact-r2,ornith122-icompact,gb10,Ornith-1.5-122B-APEX-i-compact.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.83,146077,1.0
 
24
  mellum21-12b-a2.5b-thinking-q8_0-v2-r2,mellum21-12b-a2.5b-thinking-q8_0-v2,gb10,Mellum2.1-12B-A2.5B-Thinking-Q8_0.gguf,25/25,4/4,8/9,3/3,40/41,1,40,1.53,255666,0.9934
25
  minicpm5-2b-q8,minicpm5-2b-q8,gb10,MiniCPM5-2B-Q8_0.gguf,23/23,3/3,6/6,3/3,35/35,7,35,1.9,390175,1.0
26
  nemotron35-bolt-74b-nvidia24gb-oc2-r1,nemotron35-bolt-74b-nvidia24gb-oc2,gb10,NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf,24/24,5/5,7/7,3/3,39/39,3,39,2.74,164349,1.0
27
+ nemotron35-super-vl-iquality-oc2-r1,nemotron35-super-vl-iquality-oc2,gb10,NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,3.07,119938,1.0
28
  nemotron3super-iquality-oc2-r1,nemotron3super-iquality-oc2,gb10,Nemotron-3-Super-120B-A12B-APEX-i-quality.gguf,25/25,5/5,9/9,3/3,42/42,0,42,3.4,165619,1.0
29
  ornith122-icompact-r1,ornith122-icompact,gb10,Ornith-1.5-122B-APEX-i-compact.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.79,132233,1.0
30
  ornith122-icompact-r2,ornith122-icompact,gb10,Ornith-1.5-122B-APEX-i-compact.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.83,146077,1.0