| { |
| "complete": true, |
| "checked_utc": "2026-10-06T15:59:45.382575+00:00", |
| "scope": "all", |
| "split": "confirmation", |
| "backend": "FP4 packed weights, BF16 activations/GEMM, Triton per-matrix dequantization on AMD", |
| "same_native_prompt_readout_temperature_API": true, |
| "hardware_speed_validated": false, |
| "full_decision_index": null, |
| "overall": { |
| "n": 544, |
| "agreement_percent": 97.24264705882354, |
| "bf16_accuracy_percent": 77.57352941176471, |
| "fp4_accuracy_percent": 76.83823529411765, |
| "mean_KL_bf16_to_fp4": 0.011581205562651279 |
| }, |
| "by_domain": { |
| "ESCI": { |
| "n": 32, |
| "agreement_percent": 96.875, |
| "bf16_accuracy_percent": 59.375, |
| "fp4_accuracy_percent": 56.25, |
| "mean_KL_bf16_to_fp4": 0.008368359788850645 |
| }, |
| "HoVer": { |
| "n": 32, |
| "agreement_percent": 93.75, |
| "bf16_accuracy_percent": 81.25, |
| "fp4_accuracy_percent": 75.0, |
| "mean_KL_bf16_to_fp4": 0.023173074279276965 |
| }, |
| "Habermas": { |
| "n": 32, |
| "agreement_percent": 100.0, |
| "bf16_accuracy_percent": 31.25, |
| "fp4_accuracy_percent": 31.25, |
| "mean_KL_bf16_to_fp4": 0.02629049770095534 |
| }, |
| "When2Call": { |
| "n": 32, |
| "agreement_percent": 100.0, |
| "bf16_accuracy_percent": 93.75, |
| "fp4_accuracy_percent": 93.75, |
| "mean_KL_bf16_to_fp4": 0.002222273564954566 |
| }, |
| "sarcasm_ar": { |
| "n": 32, |
| "agreement_percent": 90.625, |
| "bf16_accuracy_percent": 62.5, |
| "fp4_accuracy_percent": 59.375, |
| "mean_KL_bf16_to_fp4": 0.012054604471233589 |
| }, |
| "old_general/choice": { |
| "n": 32, |
| "agreement_percent": 100.0, |
| "bf16_accuracy_percent": 84.375, |
| "fp4_accuracy_percent": 84.375, |
| "mean_KL_bf16_to_fp4": 0.002750536940483584 |
| }, |
| "guard/humor": { |
| "n": 32, |
| "agreement_percent": 93.75, |
| "bf16_accuracy_percent": 40.625, |
| "fp4_accuracy_percent": 40.625, |
| "mean_KL_bf16_to_fp4": 0.027150870865254688 |
| }, |
| "Phishing": { |
| "n": 32, |
| "agreement_percent": 100.0, |
| "bf16_accuracy_percent": 96.875, |
| "fp4_accuracy_percent": 96.875, |
| "mean_KL_bf16_to_fp4": 0.006934639060563293 |
| }, |
| "old_general/noul": { |
| "n": 32, |
| "agreement_percent": 96.875, |
| "bf16_accuracy_percent": 93.75, |
| "fp4_accuracy_percent": 90.625, |
| "mean_KL_bf16_to_fp4": 0.0035566811986626184 |
| }, |
| "ANLI": { |
| "n": 32, |
| "agreement_percent": 100.0, |
| "bf16_accuracy_percent": 84.375, |
| "fp4_accuracy_percent": 84.375, |
| "mean_KL_bf16_to_fp4": 0.009098772482953045 |
| }, |
| "VAST": { |
| "n": 32, |
| "agreement_percent": 93.75, |
| "bf16_accuracy_percent": 71.875, |
| "fp4_accuracy_percent": 68.75, |
| "mean_KL_bf16_to_fp4": 0.009012711907023834 |
| }, |
| "SATA": { |
| "n": 32, |
| "agreement_percent": 100.0, |
| "bf16_accuracy_percent": 96.875, |
| "fp4_accuracy_percent": 96.875, |
| "mean_KL_bf16_to_fp4": 0.006667878952544886 |
| }, |
| "ContractNLI": { |
| "n": 32, |
| "agreement_percent": 93.75, |
| "bf16_accuracy_percent": 87.5, |
| "fp4_accuracy_percent": 93.75, |
| "mean_KL_bf16_to_fp4": 0.005304457598214596 |
| }, |
| "ACOS": { |
| "n": 32, |
| "agreement_percent": 100.0, |
| "bf16_accuracy_percent": 84.375, |
| "fp4_accuracy_percent": 84.375, |
| "mean_KL_bf16_to_fp4": 0.011603451698692325 |
| }, |
| "RAGTruth": { |
| "n": 32, |
| "agreement_percent": 100.0, |
| "bf16_accuracy_percent": 90.625, |
| "fp4_accuracy_percent": 90.625, |
| "mean_KL_bf16_to_fp4": 0.006294842546801113 |
| }, |
| "sarcasm_en": { |
| "n": 32, |
| "agreement_percent": 93.75, |
| "bf16_accuracy_percent": 68.75, |
| "fp4_accuracy_percent": 68.75, |
| "mean_KL_bf16_to_fp4": 0.00877219076521019 |
| }, |
| "guard/science": { |
| "n": 32, |
| "agreement_percent": 100.0, |
| "bf16_accuracy_percent": 90.625, |
| "fp4_accuracy_percent": 90.625, |
| "mean_KL_bf16_to_fp4": 0.027624650743396446 |
| } |
| } |
| } |
|
|
