File size: 3,997 Bytes
c69aaec | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 | {
"complete": true,
"checked_utc": "2026-10-06T15:59:45.382575+00:00",
"scope": "all",
"split": "confirmation",
"backend": "FP4 packed weights, BF16 activations/GEMM, Triton per-matrix dequantization on AMD",
"same_native_prompt_readout_temperature_API": true,
"hardware_speed_validated": false,
"full_decision_index": null,
"overall": {
"n": 544,
"agreement_percent": 97.24264705882354,
"bf16_accuracy_percent": 77.57352941176471,
"fp4_accuracy_percent": 76.83823529411765,
"mean_KL_bf16_to_fp4": 0.011581205562651279
},
"by_domain": {
"ESCI": {
"n": 32,
"agreement_percent": 96.875,
"bf16_accuracy_percent": 59.375,
"fp4_accuracy_percent": 56.25,
"mean_KL_bf16_to_fp4": 0.008368359788850645
},
"HoVer": {
"n": 32,
"agreement_percent": 93.75,
"bf16_accuracy_percent": 81.25,
"fp4_accuracy_percent": 75.0,
"mean_KL_bf16_to_fp4": 0.023173074279276965
},
"Habermas": {
"n": 32,
"agreement_percent": 100.0,
"bf16_accuracy_percent": 31.25,
"fp4_accuracy_percent": 31.25,
"mean_KL_bf16_to_fp4": 0.02629049770095534
},
"When2Call": {
"n": 32,
"agreement_percent": 100.0,
"bf16_accuracy_percent": 93.75,
"fp4_accuracy_percent": 93.75,
"mean_KL_bf16_to_fp4": 0.002222273564954566
},
"sarcasm_ar": {
"n": 32,
"agreement_percent": 90.625,
"bf16_accuracy_percent": 62.5,
"fp4_accuracy_percent": 59.375,
"mean_KL_bf16_to_fp4": 0.012054604471233589
},
"old_general/choice": {
"n": 32,
"agreement_percent": 100.0,
"bf16_accuracy_percent": 84.375,
"fp4_accuracy_percent": 84.375,
"mean_KL_bf16_to_fp4": 0.002750536940483584
},
"guard/humor": {
"n": 32,
"agreement_percent": 93.75,
"bf16_accuracy_percent": 40.625,
"fp4_accuracy_percent": 40.625,
"mean_KL_bf16_to_fp4": 0.027150870865254688
},
"Phishing": {
"n": 32,
"agreement_percent": 100.0,
"bf16_accuracy_percent": 96.875,
"fp4_accuracy_percent": 96.875,
"mean_KL_bf16_to_fp4": 0.006934639060563293
},
"old_general/noul": {
"n": 32,
"agreement_percent": 96.875,
"bf16_accuracy_percent": 93.75,
"fp4_accuracy_percent": 90.625,
"mean_KL_bf16_to_fp4": 0.0035566811986626184
},
"ANLI": {
"n": 32,
"agreement_percent": 100.0,
"bf16_accuracy_percent": 84.375,
"fp4_accuracy_percent": 84.375,
"mean_KL_bf16_to_fp4": 0.009098772482953045
},
"VAST": {
"n": 32,
"agreement_percent": 93.75,
"bf16_accuracy_percent": 71.875,
"fp4_accuracy_percent": 68.75,
"mean_KL_bf16_to_fp4": 0.009012711907023834
},
"SATA": {
"n": 32,
"agreement_percent": 100.0,
"bf16_accuracy_percent": 96.875,
"fp4_accuracy_percent": 96.875,
"mean_KL_bf16_to_fp4": 0.006667878952544886
},
"ContractNLI": {
"n": 32,
"agreement_percent": 93.75,
"bf16_accuracy_percent": 87.5,
"fp4_accuracy_percent": 93.75,
"mean_KL_bf16_to_fp4": 0.005304457598214596
},
"ACOS": {
"n": 32,
"agreement_percent": 100.0,
"bf16_accuracy_percent": 84.375,
"fp4_accuracy_percent": 84.375,
"mean_KL_bf16_to_fp4": 0.011603451698692325
},
"RAGTruth": {
"n": 32,
"agreement_percent": 100.0,
"bf16_accuracy_percent": 90.625,
"fp4_accuracy_percent": 90.625,
"mean_KL_bf16_to_fp4": 0.006294842546801113
},
"sarcasm_en": {
"n": 32,
"agreement_percent": 93.75,
"bf16_accuracy_percent": 68.75,
"fp4_accuracy_percent": 68.75,
"mean_KL_bf16_to_fp4": 0.00877219076521019
},
"guard/science": {
"n": 32,
"agreement_percent": 100.0,
"bf16_accuracy_percent": 90.625,
"fp4_accuracy_percent": 90.625,
"mean_KL_bf16_to_fp4": 0.027624650743396446
}
}
}
|