add Flint 5bpw (42/42) to results table and plots
Browse files- .gitattributes +1 -0
- README.md +1 -0
- models.json +20 -0
- plots/perf_per_byte.png +2 -2
- plots/tokens_per_task_vs_active.png +0 -0
- plots/wall_vs_active.png +0 -0
- plots/wall_vs_tokens.png +0 -0
- plots/wall_vs_total.png +0 -0
- results.json +53 -0
- results_full42.csv +1 -0
.gitattributes
CHANGED
|
@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
plots/perf_per_byte.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
plots/perf_per_byte.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
plots/tokens_per_task_vs_active.png filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -138,6 +138,7 @@ those as 39/42, 32/42, or 35/42 would count harness limits as model failures.
|
|
| 138 |
| [Spark-X2.5-4B](https://huggingface.co/Myric/Spark-X2.5-4B-GGUF) | Q6_K | 22/22 | 3/3 | 2/2 | 3/3 | **30/30** | 12 | 12,916 | 16.2 | 1 |
|
| 139 |
| [MiniCPM5-2B](https://huggingface.co/openbmb/MiniCPM5-2B-GGUF) | Q8_0 | 23/23 | 3/3 | 6/6 | 3/3 | **35/35** | 7 | 11,148 | 57.1 | 1 |
|
| 140 |
| [LFM3-1.7B](https://huggingface.co/LiquidAI/LFM3-1.7B-GGUF) | Q4_0 | 23/24 | 3/4 | 4/7 | 2/3 | **32/38** | 4 | 18,431 | 85.3 | 1 |
|
|
|
|
| 141 |
<!-- TABLE:END -->
|
| 142 |
|
| 143 |
Every row links to the public card for the exact quant where one exists; where the tested GGUF is
|
|
|
|
| 138 |
| [Spark-X2.5-4B](https://huggingface.co/Myric/Spark-X2.5-4B-GGUF) | Q6_K | 22/22 | 3/3 | 2/2 | 3/3 | **30/30** | 12 | 12,916 | 16.2 | 1 |
|
| 139 |
| [MiniCPM5-2B](https://huggingface.co/openbmb/MiniCPM5-2B-GGUF) | Q8_0 | 23/23 | 3/3 | 6/6 | 3/3 | **35/35** | 7 | 11,148 | 57.1 | 1 |
|
| 140 |
| [LFM3-1.7B](https://huggingface.co/LiquidAI/LFM3-1.7B-GGUF) | Q4_0 | 23/24 | 3/4 | 4/7 | 2/3 | **32/38** | 4 | 18,431 | 85.3 | 1 |
|
| 141 |
+
| [Flint](https://huggingface.co/Myric/Flint-APEX-GGUF) | APEX 5bpw | 25/25 | 5/5 | 9/9 | 3/3 | **42/42** | 0 | 4,397 | 40.1 | 1 |
|
| 142 |
<!-- TABLE:END -->
|
| 143 |
|
| 144 |
Every row links to the public card for the exact quant where one exists; where the tested GGUF is
|
models.json
CHANGED
|
@@ -531,6 +531,26 @@
|
|
| 531 |
6,
|
| 532 |
6
|
| 533 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 534 |
}
|
| 535 |
]
|
| 536 |
}
|
|
|
|
| 531 |
6,
|
| 532 |
6
|
| 533 |
]
|
| 534 |
+
},
|
| 535 |
+
{
|
| 536 |
+
"group": "flint-5bpw",
|
| 537 |
+
"model": "Flint",
|
| 538 |
+
"quant": "APEX 5bpw",
|
| 539 |
+
"active_b": 2.5,
|
| 540 |
+
"total_b": 39.8,
|
| 541 |
+
"link": "https://huggingface.co/Myric/Flint-APEX-GGUF",
|
| 542 |
+
"box": "gb10",
|
| 543 |
+
"annotate": true,
|
| 544 |
+
"short": "Flint",
|
| 545 |
+
"offset": [
|
| 546 |
+
8,
|
| 547 |
+
8
|
| 548 |
+
],
|
| 549 |
+
"size_gb": 23.7,
|
| 550 |
+
"size_offset": [
|
| 551 |
+
8,
|
| 552 |
+
8
|
| 553 |
+
]
|
| 554 |
}
|
| 555 |
]
|
| 556 |
}
|
plots/perf_per_byte.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
plots/tokens_per_task_vs_active.png
CHANGED
|
|
Git LFS Details
|
plots/wall_vs_active.png
CHANGED
|
|
plots/wall_vs_tokens.png
CHANGED
|
|
plots/wall_vs_total.png
CHANGED
|
|
results.json
CHANGED
|
@@ -73,6 +73,59 @@
|
|
| 73 |
"total_wall_s": 22017.5,
|
| 74 |
"total_tokens_out": 474702
|
| 75 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
{
|
| 77 |
"label": "gemma4-12b-qat-google-r1",
|
| 78 |
"group": "gemma4-12b-qat-google",
|
|
|
|
| 73 |
"total_wall_s": 22017.5,
|
| 74 |
"total_tokens_out": 474702
|
| 75 |
},
|
| 76 |
+
{
|
| 77 |
+
"label": "flint-5bpw",
|
| 78 |
+
"group": "flint-5bpw",
|
| 79 |
+
"box": "gb10",
|
| 80 |
+
"model_path": "Recursive-Flint-APEX-5bpw.gguf",
|
| 81 |
+
"suites": {
|
| 82 |
+
"general": {
|
| 83 |
+
"tasks": 25,
|
| 84 |
+
"valid": 25,
|
| 85 |
+
"passed": 25,
|
| 86 |
+
"void": 0,
|
| 87 |
+
"partial": 1.0,
|
| 88 |
+
"wall_s": 983.7,
|
| 89 |
+
"tokens_out": 30583,
|
| 90 |
+
"source": "results_flint-5bpw_opencode_tasks.json"
|
| 91 |
+
},
|
| 92 |
+
"frontier": {
|
| 93 |
+
"tasks": 9,
|
| 94 |
+
"valid": 9,
|
| 95 |
+
"passed": 9,
|
| 96 |
+
"void": 0,
|
| 97 |
+
"partial": 1.0,
|
| 98 |
+
"wall_s": 3136.0,
|
| 99 |
+
"tokens_out": 134212,
|
| 100 |
+
"source": "results_flint-5bpw_opencode_tasks_frontier.json"
|
| 101 |
+
},
|
| 102 |
+
"tier4a": {
|
| 103 |
+
"tasks": 3,
|
| 104 |
+
"valid": 3,
|
| 105 |
+
"passed": 3,
|
| 106 |
+
"void": 0,
|
| 107 |
+
"partial": 1.0,
|
| 108 |
+
"wall_s": 108.0,
|
| 109 |
+
"tokens_out": 3145,
|
| 110 |
+
"source": "results_flint-5bpw_opencode_tasks_tier4a.json"
|
| 111 |
+
},
|
| 112 |
+
"hard": {
|
| 113 |
+
"tasks": 5,
|
| 114 |
+
"valid": 5,
|
| 115 |
+
"passed": 5,
|
| 116 |
+
"void": 0,
|
| 117 |
+
"partial": 1.0,
|
| 118 |
+
"wall_s": 380.5,
|
| 119 |
+
"tokens_out": 16754,
|
| 120 |
+
"source": "results_flint-5bpw_opencode_tasks_hard.json"
|
| 121 |
+
}
|
| 122 |
+
},
|
| 123 |
+
"total_passed": 42,
|
| 124 |
+
"total_valid": 42,
|
| 125 |
+
"total_void": 0,
|
| 126 |
+
"total_wall_s": 4608.2,
|
| 127 |
+
"total_tokens_out": 184694
|
| 128 |
+
},
|
| 129 |
{
|
| 130 |
"label": "gemma4-12b-qat-google-r1",
|
| 131 |
"group": "gemma4-12b-qat-google",
|
results_full42.csv
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
label,group,box,model_path,general,hard,frontier,tier4a,valid_total,void,passed_total,wall_h,tokens_out,tokens_per_task,tok_s,partial_mean
|
| 2 |
bonsai2-pq2-oc2-r1,bonsai2-pq2-oc2,gb10,Ternary-Bonsai-2-27B-PQ2_0.gguf,25/25,5/5,8/8,3/3,41/41,1,41,6.12,474702,11578.1,21.56,1.0
|
|
|
|
| 3 |
gemma4-12b-qat-google-r1,gemma4-12b-qat-google,gb10,gemma-4-12b-it-qat-q4_0.gguf,24/24,5/5,7/7,3/3,39/39,3,39,7.86,379608,9733.5,13.41,1.0
|
| 4 |
gemma4-12b-qat-google-r2,gemma4-12b-qat-google,gb10,gemma-4-12b-it-qat-q4_0.gguf,24/25,5/5,8/8,3/3,40/41,1,40,6.92,312468,7621.2,12.54,0.9756
|
| 5 |
gemma4-12b-qat-grid-r1,gemma4-12b-qat-grid,gb10,gemma-4-12b-it-qat-grid-q4_0-all.gguf,24/25,5/5,9/9,3/3,41/42,0,41,5.12,249114,5931.3,13.52,0.9762
|
|
|
|
| 1 |
label,group,box,model_path,general,hard,frontier,tier4a,valid_total,void,passed_total,wall_h,tokens_out,tokens_per_task,tok_s,partial_mean
|
| 2 |
bonsai2-pq2-oc2-r1,bonsai2-pq2-oc2,gb10,Ternary-Bonsai-2-27B-PQ2_0.gguf,25/25,5/5,8/8,3/3,41/41,1,41,6.12,474702,11578.1,21.56,1.0
|
| 3 |
+
flint-5bpw,flint-5bpw,gb10,Recursive-Flint-APEX-5bpw.gguf,25/25,5/5,9/9,3/3,42/42,0,42,1.28,184694,4397.5,40.08,1.0
|
| 4 |
gemma4-12b-qat-google-r1,gemma4-12b-qat-google,gb10,gemma-4-12b-it-qat-q4_0.gguf,24/24,5/5,7/7,3/3,39/39,3,39,7.86,379608,9733.5,13.41,1.0
|
| 5 |
gemma4-12b-qat-google-r2,gemma4-12b-qat-google,gb10,gemma-4-12b-it-qat-q4_0.gguf,24/25,5/5,8/8,3/3,40/41,1,40,6.92,312468,7621.2,12.54,0.9756
|
| 6 |
gemma4-12b-qat-grid-r1,gemma4-12b-qat-grid,gb10,gemma-4-12b-it-qat-grid-q4_0-all.gguf,24/25,5/5,9/9,3/3,41/42,0,41,5.12,249114,5931.3,13.52,0.9762
|