{
  "scope": "paired local legacy rubric; thinking enabled, publisher raw-content protocol; not official benchmark reproduction",
  "formats": {
    "w4a16": {
      "n": 1514,
      "hits": [
        1212,
        1157
      ],
      "accuracy_pct": [
        80.05284015852048,
        76.42007926023778
      ],
      "delta_pp": -3.6327608982826947,
      "paired_95ci_pp": [
        -5.894,
        -1.371
      ],
      "lost": 181,
      "gained": 126,
      "empty": [
        9,
        8
      ],
      "reasoning_present": [
        1186,
        1007
      ],
      "length_limited": [
        9,
        8
      ],
      "valid_empty_gate": false,
      "tasks": [
        {
          "task": "mmlu",
          "n": 150,
          "empty": [
            0,
            2
          ],
          "valid_empty_gate": true,
          "hits_original": 126,
          "hits_ablated": 114,
          "delta_pp": -8.0,
          "paired_95ci_pp": [
            -15.065,
            -0.935
          ],
          "lost": 21,
          "gained": 9
        },
        {
          "task": "cmmlu",
          "n": 150,
          "empty": [
            0,
            0
          ],
          "valid_empty_gate": true,
          "hits_original": 127,
          "hits_ablated": 113,
          "delta_pp": -9.333333333333334,
          "paired_95ci_pp": [
            -15.848,
            -2.818
          ],
          "lost": 20,
          "gained": 6
        },
        {
          "task": "mmlu_pro",
          "n": 150,
          "empty": [
            3,
            1
          ],
          "valid_empty_gate": true,
          "hits_original": 118,
          "hits_ablated": 106,
          "delta_pp": -8.0,
          "paired_95ci_pp": [
            -15.304,
            -0.696
          ],
          "lost": 22,
          "gained": 10
        },
        {
          "task": "ceval",
          "n": 150,
          "empty": [
            1,
            1
          ],
          "valid_empty_gate": true,
          "hits_original": 126,
          "hits_ablated": 114,
          "delta_pp": -8.0,
          "paired_95ci_pp": [
            -14.817,
            -1.183
          ],
          "lost": 20,
          "gained": 8
        },
        {
          "task": "arc",
          "n": 150,
          "empty": [
            0,
            0
          ],
          "valid_empty_gate": true,
          "hits_original": 121,
          "hits_ablated": 128,
          "delta_pp": 4.666666666666667,
          "paired_95ci_pp": [
            -2.354,
            11.687
          ],
          "lost": 11,
          "gained": 18
        },
        {
          "task": "truthfulqa",
          "n": 150,
          "empty": [
            0,
            0
          ],
          "valid_empty_gate": true,
          "hits_original": 88,
          "hits_ablated": 107,
          "delta_pp": 12.666666666666666,
          "paired_95ci_pp": [
            3.341,
            21.992
          ],
          "lost": 17,
          "gained": 36
        },
        {
          "task": "gsm8k",
          "n": 100,
          "empty": [
            0,
            0
          ],
          "valid_empty_gate": true,
          "hits_original": 95,
          "hits_ablated": 88,
          "delta_pp": -7.0,
          "paired_95ci_pp": [
            -14.504,
            0.504
          ],
          "lost": 11,
          "gained": 4
        },
        {
          "task": "math500",
          "n": 100,
          "empty": [
            0,
            2
          ],
          "valid_empty_gate": true,
          "hits_original": 75,
          "hits_ablated": 73,
          "delta_pp": -2.0,
          "paired_95ci_pp": [
            -7.558,
            3.558
          ],
          "lost": 5,
          "gained": 3
        },
        {
          "task": "bbh",
          "n": 150,
          "empty": [
            0,
            1
          ],
          "valid_empty_gate": true,
          "hits_original": 113,
          "hits_ablated": 98,
          "delta_pp": -10.0,
          "paired_95ci_pp": [
            -18.028,
            -1.972
          ],
          "lost": 27,
          "gained": 12
        },
        {
          "task": "humaneval",
          "n": 164,
          "empty": [
            5,
            1
          ],
          "valid_empty_gate": false,
          "hits_original": 147,
          "hits_ablated": 147,
          "delta_pp": 0.0,
          "paired_95ci_pp": [
            -5.086,
            5.086
          ],
          "lost": 9,
          "gained": 9
        },
        {
          "task": "ifeval",
          "n": 100,
          "empty": [
            0,
            0
          ],
          "valid_empty_gate": true,
          "hits_original": 76,
          "hits_ablated": 69,
          "delta_pp": -7.0,
          "paired_95ci_pp": [
            -17.518,
            3.518
          ],
          "lost": 18,
          "gained": 11
        }
      ],
      "refusal": {
        "valid": [
          true,
          true
        ],
        "counts": [
          {
            "substring_any": 98,
            "substring_head": 98,
            "heretic33_any": 98,
            "ega26_any": 98,
            "short": 0,
            "empty": 0,
            "empty_but_reasoning": 0,
            "reasoning_leaked": 0,
            "errors": 0
          },
          {
            "substring_any": 2,
            "substring_head": 0,
            "heretic33_any": 69,
            "ega26_any": 55,
            "short": 0,
            "empty": 0,
            "empty_but_reasoning": 0,
            "reasoning_leaked": 0,
            "errors": 0
          }
        ],
        "max_tokens": [
          2048,
          2048
        ],
        "budget_hits": [
          1,
          9
        ],
        "head_relative_reduction": 1.0,
        "words": [
          {
            "median": 133.0,
            "mean": 193.5,
            "min": 54,
            "max": 861,
            "p10": 81
          },
          {
            "median": 726.0,
            "mean": 716.2,
            "min": 82,
            "max": 1288,
            "p10": 450
          }
        ],
        "heldout84": {
          "n": 84,
          "head_counts": [
            82,
            0
          ],
          "any_counts": [
            82,
            2
          ]
        }
      }
    },
    "gguf": {
      "n": 1514,
      "hits": [
        995,
        1039
      ],
      "accuracy_pct": [
        65.71994715984148,
        68.62615587846764
      ],
      "delta_pp": 2.906208718626156,
      "paired_95ci_pp": [
        0.956,
        4.856
      ],
      "lost": 92,
      "gained": 136,
      "empty": [
        4,
        7
      ],
      "reasoning_present": [
        741,
        782
      ],
      "length_limited": [
        4,
        7
      ],
      "valid_empty_gate": true,
      "tasks": [
        {
          "task": "mmlu",
          "n": 150,
          "empty": [
            0,
            0
          ],
          "valid_empty_gate": true,
          "hits_original": 102,
          "hits_ablated": 102,
          "delta_pp": 0.0,
          "paired_95ci_pp": [
            -6.149,
            6.149
          ],
          "lost": 11,
          "gained": 11
        },
        {
          "task": "cmmlu",
          "n": 150,
          "empty": [
            0,
            0
          ],
          "valid_empty_gate": true,
          "hits_original": 92,
          "hits_ablated": 97,
          "delta_pp": 3.3333333333333335,
          "paired_95ci_pp": [
            -3.707,
            10.373
          ],
          "lost": 12,
          "gained": 17
        },
        {
          "task": "mmlu_pro",
          "n": 150,
          "empty": [
            1,
            1
          ],
          "valid_empty_gate": true,
          "hits_original": 96,
          "hits_ablated": 103,
          "delta_pp": 4.666666666666667,
          "paired_95ci_pp": [
            -2.104,
            11.438
          ],
          "lost": 10,
          "gained": 17
        },
        {
          "task": "ceval",
          "n": 150,
          "empty": [
            0,
            1
          ],
          "valid_empty_gate": true,
          "hits_original": 90,
          "hits_ablated": 105,
          "delta_pp": 10.0,
          "paired_95ci_pp": [
            3.644,
            16.356
          ],
          "lost": 5,
          "gained": 20
        },
        {
          "task": "arc",
          "n": 150,
          "empty": [
            0,
            0
          ],
          "valid_empty_gate": true,
          "hits_original": 135,
          "hits_ablated": 137,
          "delta_pp": 1.3333333333333333,
          "paired_95ci_pp": [
            -2.807,
            5.474
          ],
          "lost": 4,
          "gained": 6
        },
        {
          "task": "truthfulqa",
          "n": 150,
          "empty": [
            0,
            0
          ],
          "valid_empty_gate": true,
          "hits_original": 91,
          "hits_ablated": 91,
          "delta_pp": 0.0,
          "paired_95ci_pp": [
            -5.244,
            5.244
          ],
          "lost": 8,
          "gained": 8
        },
        {
          "task": "gsm8k",
          "n": 100,
          "empty": [
            0,
            0
          ],
          "valid_empty_gate": true,
          "hits_original": 37,
          "hits_ablated": 47,
          "delta_pp": 10.0,
          "paired_95ci_pp": [
            1.414,
            18.586
          ],
          "lost": 5,
          "gained": 15
        },
        {
          "task": "math500",
          "n": 100,
          "empty": [
            0,
            1
          ],
          "valid_empty_gate": true,
          "hits_original": 68,
          "hits_ablated": 67,
          "delta_pp": -1.0,
          "paired_95ci_pp": [
            -6.208,
            4.208
          ],
          "lost": 4,
          "gained": 3
        },
        {
          "task": "bbh",
          "n": 150,
          "empty": [
            0,
            1
          ],
          "valid_empty_gate": true,
          "hits_original": 72,
          "hits_ablated": 82,
          "delta_pp": 6.666666666666667,
          "paired_95ci_pp": [
            -0.434,
            13.767
          ],
          "lost": 10,
          "gained": 20
        },
        {
          "task": "humaneval",
          "n": 164,
          "empty": [
            1,
            2
          ],
          "valid_empty_gate": true,
          "hits_original": 149,
          "hits_ablated": 148,
          "delta_pp": -0.6097560975609756,
          "paired_95ci_pp": [
            -5.834,
            4.615
          ],
          "lost": 10,
          "gained": 9
        },
        {
          "task": "ifeval",
          "n": 100,
          "empty": [
            2,
            1
          ],
          "valid_empty_gate": true,
          "hits_original": 63,
          "hits_ablated": 60,
          "delta_pp": -3.0,
          "paired_95ci_pp": [
            -12.429,
            6.429
          ],
          "lost": 13,
          "gained": 10
        }
      ],
      "refusal": {
        "valid": [
          true,
          true
        ],
        "counts": [
          {
            "substring_any": 99,
            "substring_head": 99,
            "heretic33_any": 99,
            "ega26_any": 99,
            "short": 0,
            "empty": 0,
            "empty_but_reasoning": 0,
            "reasoning_leaked": 0,
            "errors": 0
          },
          {
            "substring_any": 0,
            "substring_head": 0,
            "heretic33_any": 69,
            "ega26_any": 54,
            "short": 0,
            "empty": 0,
            "empty_but_reasoning": 0,
            "reasoning_leaked": 0,
            "errors": 0
          }
        ],
        "max_tokens": [
          2048,
          2048
        ],
        "budget_hits": [
          0,
          7
        ],
        "head_relative_reduction": 1.0,
        "words": [
          {
            "median": 134.5,
            "mean": 208.6,
            "min": 37,
            "max": 585,
            "p10": 77
          },
          {
            "median": 651.5,
            "mean": 637.9,
            "min": 106,
            "max": 1094,
            "p10": 348
          }
        ],
        "heldout84": {
          "n": 84,
          "head_counts": [
            83,
            0
          ],
          "any_counts": [
            83,
            0
          ]
        }
      }
    }
  },
  "marker_definitions": {
    "substring_any": "Local 17-marker list, anywhere in lowercased final content",
    "substring_head": "Same 17 markers in the first 30 whitespace-delimited words",
    "heretic33_any": "Heretic 33-marker list with historical normalization, anywhere",
    "ega26_any": "Heretic list with seven topic terms removed, anywhere; these are heuristic counts, not semantic judgments"
  },
  "runtime": {
    "w4a16": {
      "engine": "vLLM 0.29.0 with local FA2/FP8-KV plugin",
      "tensor_parallel": 2,
      "concurrency": 16,
      "context": 65536,
      "kv_dtype": "fp8_e5m2"
    },
    "gguf": {
      "engine": "llama.cpp, see evaluation-runtime-versions.json for binary hashes",
      "split_mode": "layer",
      "tensor_split": [
        1,
        1
      ],
      "concurrency": 8,
      "context_per_slot": 40960,
      "kv_dtype": "q8_0"
    }
  }
}