development-report.json

{
  "complete": true,
  "checked_utc": "2026-10-06T15:57:51.614032+00:00",
  "scope": "all",
  "split": "development",
  "backend": "FP4 packed weights, BF16 activations/GEMM, Triton per-matrix dequantization on AMD",
  "same_native_prompt_readout_temperature_API": true,
  "hardware_speed_validated": false,
  "full_decision_index": null,
  "overall": {
    "n": 608,
    "agreement_percent": 95.88815789473684,
    "bf16_accuracy_percent": 80.92105263157895,
    "fp4_accuracy_percent": 81.57894736842105,
    "mean_KL_bf16_to_fp4": 0.015593464629358985
  },
  "by_domain": {
    "ACOS": {
      "n": 32,
      "agreement_percent": 93.75,
      "bf16_accuracy_percent": 78.125,
      "fp4_accuracy_percent": 84.375,
      "mean_KL_bf16_to_fp4": 0.013701164049475172
    },
    "When2Call": {
      "n": 32,
      "agreement_percent": 100.0,
      "bf16_accuracy_percent": 93.75,
      "fp4_accuracy_percent": 93.75,
      "mean_KL_bf16_to_fp4": 0.0016987235865958965
    },
    "guard/mmlu_development": {
      "n": 32,
      "agreement_percent": 96.875,
      "bf16_accuracy_percent": 68.75,
      "fp4_accuracy_percent": 71.875,
      "mean_KL_bf16_to_fp4": 0.04259847080039246
    },
    "ContractNLI": {
      "n": 32,
      "agreement_percent": 100.0,
      "bf16_accuracy_percent": 93.75,
      "fp4_accuracy_percent": 93.75,
      "mean_KL_bf16_to_fp4": 0.010235464961968524
    },
    "VAST": {
      "n": 32,
      "agreement_percent": 96.875,
      "bf16_accuracy_percent": 90.625,
      "fp4_accuracy_percent": 87.5,
      "mean_KL_bf16_to_fp4": 0.013530445497465765
    },
    "sarcasm_en": {
      "n": 32,
      "agreement_percent": 96.875,
      "bf16_accuracy_percent": 68.75,
      "fp4_accuracy_percent": 65.625,
      "mean_KL_bf16_to_fp4": 0.010841849286577542
    },
    "ANLI": {
      "n": 32,
      "agreement_percent": 90.625,
      "bf16_accuracy_percent": 81.25,
      "fp4_accuracy_percent": 87.5,
      "mean_KL_bf16_to_fp4": 0.007467320146816308
    },
    "HoVer": {
      "n": 32,
      "agreement_percent": 100.0,
      "bf16_accuracy_percent": 96.875,
      "fp4_accuracy_percent": 96.875,
      "mean_KL_bf16_to_fp4": 0.010262760266491694
    },
    "RAGTruth": {
      "n": 32,
      "agreement_percent": 100.0,
      "bf16_accuracy_percent": 93.75,
      "fp4_accuracy_percent": 93.75,
      "mean_KL_bf16_to_fp4": 0.0026213688997479747
    },
    "ESCI": {
      "n": 32,
      "agreement_percent": 96.875,
      "bf16_accuracy_percent": 53.125,
      "fp4_accuracy_percent": 53.125,
      "mean_KL_bf16_to_fp4": 0.013929468238803712
    },
    "guard/science": {
      "n": 32,
      "agreement_percent": 100.0,
      "bf16_accuracy_percent": 96.875,
      "fp4_accuracy_percent": 96.875,
      "mean_KL_bf16_to_fp4": 0.00532867118309089
    },
    "guard/solver_development": {
      "n": 32,
      "agreement_percent": 96.875,
      "bf16_accuracy_percent": 93.75,
      "fp4_accuracy_percent": 90.625,
      "mean_KL_bf16_to_fp4": 0.009307941474104636
    },
    "guard/chess_development": {
      "n": 32,
      "agreement_percent": 84.375,
      "bf16_accuracy_percent": 53.125,
      "fp4_accuracy_percent": 46.875,
      "mean_KL_bf16_to_fp4": 0.04428748497978523
    },
    "Habermas": {
      "n": 32,
      "agreement_percent": 75.0,
      "bf16_accuracy_percent": 21.875,
      "fp4_accuracy_percent": 40.625,
      "mean_KL_bf16_to_fp4": 0.041353885293041556
    },
    "guard/broad_development": {
      "n": 32,
      "agreement_percent": 100.0,
      "bf16_accuracy_percent": 90.625,
      "fp4_accuracy_percent": 90.625,
      "mean_KL_bf16_to_fp4": 0.016964693950754887
    },
    "SATA": {
      "n": 32,
      "agreement_percent": 100.0,
      "bf16_accuracy_percent": 93.75,
      "fp4_accuracy_percent": 93.75,
      "mean_KL_bf16_to_fp4": 0.005437098679933294
    },
    "Phishing": {
      "n": 32,
      "agreement_percent": 100.0,
      "bf16_accuracy_percent": 100.0,
      "fp4_accuracy_percent": 100.0,
      "mean_KL_bf16_to_fp4": 0.0025803647593062337
    },
    "guard/general_development": {
      "n": 32,
      "agreement_percent": 100.0,
      "bf16_accuracy_percent": 87.5,
      "fp4_accuracy_percent": 87.5,
      "mean_KL_bf16_to_fp4": 0.003838940957747038
    },
    "sarcasm_ar": {
      "n": 32,
      "agreement_percent": 93.75,
      "bf16_accuracy_percent": 81.25,
      "fp4_accuracy_percent": 75.0,
      "mean_KL_bf16_to_fp4": 0.040289710945721875
    }
  }
}