| { |
| "model_line": "openjev-e4b", |
| "release": "1.0", |
| "benchmark_artifact": "local NF4 frozen base + dense adapted layers + LoRA + decision head", |
| "source_summary": "eval/raw_run/summary.json", |
| "raw_control_summary": "eval/raw_e4b_nf4_control_summary.json", |
| "tasks": { |
| "arc_challenge": { |
| "status": "measured", |
| "display": "ARC-Challenge", |
| "n": 1172, |
| "accuracy": 0.878839590443686, |
| "nll": 0.3434354264088279, |
| "brier": 0.18004018282065612, |
| "ece": 0.018640702693919233, |
| "p50_ms": 173.45179200128769, |
| "p95_ms": 241.31601209974178, |
| "raw_e4b_nf4_accuracy": 0.8737, |
| "delta_vs_raw_pp": 0.5139590443685949 |
| }, |
| "winogrande": { |
| "status": "measured", |
| "display": "WinoGrande", |
| "n": 1267, |
| "accuracy": 0.664561957379637, |
| "nll": 0.8732282598352692, |
| "brier": 0.5079363220206212, |
| "ece": 0.1986676781891346, |
| "p50_ms": 168.07529400102794, |
| "p95_ms": 241.4132092009823, |
| "raw_e4b_nf4_accuracy": 0.6014, |
| "delta_vs_raw_pp": 6.316195737963692 |
| }, |
| "arc_easy": { |
| "status": "measured", |
| "display": "ARC-Easy", |
| "n": 2376, |
| "accuracy": 0.9562289562289562, |
| "nll": 0.15331558248276225, |
| "brier": 0.07269179989034444, |
| "ece": 0.029571857402955024, |
| "p50_ms": 172.40582649901626, |
| "p95_ms": 231.58336125015921, |
| "raw_e4b_nf4_accuracy": 0.9558, |
| "delta_vs_raw_pp": 0.04289562289562232 |
| }, |
| "hellaswag": { |
| "status": "measured", |
| "display": "HellaSwag", |
| "n": 10042, |
| "accuracy": 0.8977295359490142, |
| "nll": 0.2736215284007136, |
| "brier": 0.14571887516017584, |
| "ece": 0.012820075397336044, |
| "p50_ms": 230.12878749796073, |
| "p95_ms": 276.0439652007335, |
| "raw_e4b_nf4_accuracy": 0.7651, |
| "delta_vs_raw_pp": 13.262953594901417 |
| }, |
| "mmlu": { |
| "status": "measured", |
| "display": "MMLU", |
| "n": 14042, |
| "accuracy": 0.6441390115368181, |
| "nll": 0.8705675106095903, |
| "brier": 0.45396635565340543, |
| "ece": 0.03314363695835977, |
| "p50_ms": 196.04511249963252, |
| "p95_ms": 303.28189720094076, |
| "raw_e4b_nf4_accuracy": 0.6619, |
| "delta_vs_raw_pp": -1.776098846318197 |
| } |
| } |
| } |
|
|
