Hugging Face's logo

bowmanslayer
/
VeriLoop-E2-W4A16

Text Generation
Transformers
Safetensors
qwen3_5
image-text-to-text
veriloop
auto-round
community-quantization
experimental
conversational
4-bit precision
Model card Files
xet
 Community
VeriLoop-E2-W4A16 / evaluation.json
bowmanslayer's picture
bowmanslayer
Release unmodified quantization with complete saved evaluation metrics and historical reference
711ed9f verified
Raw History Blame Contribute Delete
13.5 kB
{
"scope": "paired local legacy rubric; thinking enabled, publisher raw-content protocol; not official benchmark reproduction",
"formats": {
"w4a16": {
"n": 1514,
"hits": [
1212,
1157
],
"accuracy_pct": [
80.05284015852048,
76.42007926023778
],
"delta_pp": -3.6327608982826947,
"paired_95ci_pp": [
-5.894,
-1.371
],
"lost": 181,
"gained": 126,
"empty": [
9,
8
],
"reasoning_present": [
1186,
1007
],
"length_limited": [
9,
8
],
"valid_empty_gate": false,
"tasks": [
{
"task": "mmlu",
"n": 150,
"empty": [
0,
2
],
"valid_empty_gate": true,
"hits_original": 126,
"hits_ablated": 114,
"delta_pp": -8.0,
"paired_95ci_pp": [
-15.065,
-0.935
],
"lost": 21,
"gained": 9
},
{
"task": "cmmlu",
"n": 150,
"empty": [
0,
0
],
"valid_empty_gate": true,
"hits_original": 127,
"hits_ablated": 113,
"delta_pp": -9.333333333333334,
"paired_95ci_pp": [
-15.848,
-2.818
],
"lost": 20,
"gained": 6
},
{
"task": "mmlu_pro",
"n": 150,
"empty": [
3,
1
],
"valid_empty_gate": true,
"hits_original": 118,
"hits_ablated": 106,
"delta_pp": -8.0,
"paired_95ci_pp": [
-15.304,
-0.696
],
"lost": 22,
"gained": 10
},
{
"task": "ceval",
"n": 150,
"empty": [
1,
1
],
"valid_empty_gate": true,
"hits_original": 126,
"hits_ablated": 114,
"delta_pp": -8.0,
"paired_95ci_pp": [
-14.817,
-1.183
],
"lost": 20,
"gained": 8
},
{
"task": "arc",
"n": 150,
"empty": [
0,
0
],
"valid_empty_gate": true,
"hits_original": 121,
"hits_ablated": 128,
"delta_pp": 4.666666666666667,
"paired_95ci_pp": [
-2.354,
11.687
],
"lost": 11,
"gained": 18
},
{
"task": "truthfulqa",
"n": 150,
"empty": [
0,
0
],
"valid_empty_gate": true,
"hits_original": 88,
"hits_ablated": 107,
"delta_pp": 12.666666666666666,
"paired_95ci_pp": [
3.341,
21.992
],
"lost": 17,
"gained": 36
},
{
"task": "gsm8k",
"n": 100,
"empty": [
0,
0
],
"valid_empty_gate": true,
"hits_original": 95,
"hits_ablated": 88,
"delta_pp": -7.0,
"paired_95ci_pp": [
-14.504,
0.504
],
"lost": 11,
"gained": 4
},
{
"task": "math500",
"n": 100,
"empty": [
0,
2
],
"valid_empty_gate": true,
"hits_original": 75,
"hits_ablated": 73,
"delta_pp": -2.0,
"paired_95ci_pp": [
-7.558,
3.558
],
"lost": 5,
"gained": 3
},
{
"task": "bbh",
"n": 150,
"empty": [
0,
1
],
"valid_empty_gate": true,
"hits_original": 113,
"hits_ablated": 98,
"delta_pp": -10.0,
"paired_95ci_pp": [
-18.028,
-1.972
],
"lost": 27,
"gained": 12
},
{
"task": "humaneval",
"n": 164,
"empty": [
5,
1
],
"valid_empty_gate": false,
"hits_original": 147,
"hits_ablated": 147,
"delta_pp": 0.0,
"paired_95ci_pp": [
-5.086,
5.086
],
"lost": 9,
"gained": 9
},
{
"task": "ifeval",
"n": 100,
"empty": [
0,
0
],
"valid_empty_gate": true,
"hits_original": 76,
"hits_ablated": 69,
"delta_pp": -7.0,
"paired_95ci_pp": [
-17.518,
3.518
],
"lost": 18,
"gained": 11
}
],
"refusal": {
"valid": [
true,
true
],
"counts": [
{
"substring_any": 98,
"substring_head": 98,
"heretic33_any": 98,
"ega26_any": 98,
"short": 0,
"empty": 0,
"empty_but_reasoning": 0,
"reasoning_leaked": 0,
"errors": 0
},
{
"substring_any": 2,
"substring_head": 0,
"heretic33_any": 69,
"ega26_any": 55,
"short": 0,
"empty": 0,
"empty_but_reasoning": 0,
"reasoning_leaked": 0,
"errors": 0
}
],
"max_tokens": [
2048,
2048
],
"budget_hits": [
1,
9
],
"head_relative_reduction": 1.0,
"words": [
{
"median": 133.0,
"mean": 193.5,
"min": 54,
"max": 861,
"p10": 81
},
{
"median": 726.0,
"mean": 716.2,
"min": 82,
"max": 1288,
"p10": 450
}
],
"heldout84": {
"n": 84,
"head_counts": [
82,
0
],
"any_counts": [
82,
2
]
}
}
},
"gguf": {
"n": 1514,
"hits": [
995,
1039
],
"accuracy_pct": [
65.71994715984148,
68.62615587846764
],
"delta_pp": 2.906208718626156,
"paired_95ci_pp": [
0.956,
4.856
],
"lost": 92,
"gained": 136,
"empty": [
4,
7
],
"reasoning_present": [
741,
782
],
"length_limited": [
4,
7
],
"valid_empty_gate": true,
"tasks": [
{
"task": "mmlu",
"n": 150,
"empty": [
0,
0
],
"valid_empty_gate": true,
"hits_original": 102,
"hits_ablated": 102,
"delta_pp": 0.0,
"paired_95ci_pp": [
-6.149,
6.149
],
"lost": 11,
"gained": 11
},
{
"task": "cmmlu",
"n": 150,
"empty": [
0,
0
],
"valid_empty_gate": true,
"hits_original": 92,
"hits_ablated": 97,
"delta_pp": 3.3333333333333335,
"paired_95ci_pp": [
-3.707,
10.373
],
"lost": 12,
"gained": 17
},
{
"task": "mmlu_pro",
"n": 150,
"empty": [
1,
1
],
"valid_empty_gate": true,
"hits_original": 96,
"hits_ablated": 103,
"delta_pp": 4.666666666666667,
"paired_95ci_pp": [
-2.104,
11.438
],
"lost": 10,
"gained": 17
},
{
"task": "ceval",
"n": 150,
"empty": [
0,
1
],
"valid_empty_gate": true,
"hits_original": 90,
"hits_ablated": 105,
"delta_pp": 10.0,
"paired_95ci_pp": [
3.644,
16.356
],
"lost": 5,
"gained": 20
},
{
"task": "arc",
"n": 150,
"empty": [
0,
0
],
"valid_empty_gate": true,
"hits_original": 135,
"hits_ablated": 137,
"delta_pp": 1.3333333333333333,
"paired_95ci_pp": [
-2.807,
5.474
],
"lost": 4,
"gained": 6
},
{
"task": "truthfulqa",
"n": 150,
"empty": [
0,
0
],
"valid_empty_gate": true,
"hits_original": 91,
"hits_ablated": 91,
"delta_pp": 0.0,
"paired_95ci_pp": [
-5.244,
5.244
],
"lost": 8,
"gained": 8
},
{
"task": "gsm8k",
"n": 100,
"empty": [
0,
0
],
"valid_empty_gate": true,
"hits_original": 37,
"hits_ablated": 47,
"delta_pp": 10.0,
"paired_95ci_pp": [
1.414,
18.586
],
"lost": 5,
"gained": 15
},
{
"task": "math500",
"n": 100,
"empty": [
0,
1
],
"valid_empty_gate": true,
"hits_original": 68,
"hits_ablated": 67,
"delta_pp": -1.0,
"paired_95ci_pp": [
-6.208,
4.208
],
"lost": 4,
"gained": 3
},
{
"task": "bbh",
"n": 150,
"empty": [
0,
1
],
"valid_empty_gate": true,
"hits_original": 72,
"hits_ablated": 82,
"delta_pp": 6.666666666666667,
"paired_95ci_pp": [
-0.434,
13.767
],
"lost": 10,
"gained": 20
},
{
"task": "humaneval",
"n": 164,
"empty": [
1,
2
],
"valid_empty_gate": true,
"hits_original": 149,
"hits_ablated": 148,
"delta_pp": -0.6097560975609756,
"paired_95ci_pp": [
-5.834,
4.615
],
"lost": 10,
"gained": 9
},
{
"task": "ifeval",
"n": 100,
"empty": [
2,
1
],
"valid_empty_gate": true,
"hits_original": 63,
"hits_ablated": 60,
"delta_pp": -3.0,
"paired_95ci_pp": [
-12.429,
6.429
],
"lost": 13,
"gained": 10
}
],
"refusal": {
"valid": [
true,
true
],
"counts": [
{
"substring_any": 99,
"substring_head": 99,
"heretic33_any": 99,
"ega26_any": 99,
"short": 0,
"empty": 0,
"empty_but_reasoning": 0,
"reasoning_leaked": 0,
"errors": 0
},
{
"substring_any": 0,
"substring_head": 0,
"heretic33_any": 69,
"ega26_any": 54,
"short": 0,
"empty": 0,
"empty_but_reasoning": 0,
"reasoning_leaked": 0,
"errors": 0
}
],
"max_tokens": [
2048,
2048
],
"budget_hits": [
0,
7
],
"head_relative_reduction": 1.0,
"words": [
{
"median": 134.5,
"mean": 208.6,
"min": 37,
"max": 585,
"p10": 77
},
{
"median": 651.5,
"mean": 637.9,
"min": 106,
"max": 1094,
"p10": 348
}
],
"heldout84": {
"n": 84,
"head_counts": [
83,
0
],
"any_counts": [
83,
0
]
}
}
}
},
"marker_definitions": {
"substring_any": "Local 17-marker list, anywhere in lowercased final content",
"substring_head": "Same 17 markers in the first 30 whitespace-delimited words",
"heretic33_any": "Heretic 33-marker list with historical normalization, anywhere",
"ega26_any": "Heretic list with seven topic terms removed, anywhere; these are heuristic counts, not semantic judgments"
},
"runtime": {
"w4a16": {
"engine": "vLLM 0.29.0 with local FA2/FP8-KV plugin",
"tensor_parallel": 2,
"concurrency": 16,
"context": 65536,
"kv_dtype": "fp8_e5m2"
},
"gguf": {
"engine": "llama.cpp, see evaluation-runtime-versions.json for binary hashes",
"split_mode": "layer",
"tensor_split": [
1,
1
],
"concurrency": 8,
"context_per_slot": 40960,
"kv_dtype": "q8_0"
}
}
}