{
 "model": "D1A-E4B v0.5",
 "temperature": 1.782,
 "method": "d1a.eval.benchmark through scripts/quality_gate.py (--head-run, --card-suites, --suites) on the MLX 8-bit builds of v0.4 and v0.5, held-out partitions; pr-labels:test and test-ja there cover only the records inside the training context (183 and 18). The full PR test sets were scored on the bf16 PyTorch checkpoints with the serving context.",
 "sets": {
  "devtools-v1": {
   "v0.4": {
    "n": 1074,
    "acc": 0.6378,
    "ece": 0.1119,
    "nll": 0.8652,
    "mean_conf": 0.7451
   },
   "v0.5": {
    "n": 1074,
    "acc": 0.6983,
    "ece": 0.0483,
    "nll": 0.6713,
    "mean_conf": 0.6951
   }
  },
  "documents-v1": {
   "v0.4": {
    "n": 920,
    "acc": 0.8663,
    "ece": 0.0297,
    "nll": 0.3864,
    "mean_conf": 0.8838
   },
   "v0.5": {
    "n": 920,
    "acc": 0.875,
    "ece": 0.0417,
    "nll": 0.3556,
    "mean_conf": 0.9135
   }
  },
  "hard-v1": {
   "v0.4": {
    "n": 1083,
    "acc": 0.5522,
    "ece": 0.0379,
    "nll": 1.0292,
    "mean_conf": 0.559
   },
   "v0.5": {
    "n": 1083,
    "acc": 0.7147,
    "ece": 0.0448,
    "nll": 0.6907,
    "mean_conf": 0.6707
   }
  },
  "ja-jglue_development": {
   "v0.4": {
    "n": 1500,
    "acc": 0.81,
    "ece": 0.0299,
    "nll": 0.4757,
    "mean_conf": 0.7831
   },
   "v0.5": {
    "n": 1500,
    "acc": 0.8207,
    "ece": 0.0263,
    "nll": 0.4357,
    "mean_conf": 0.7996
   }
  },
  "pr-labels_test": {
   "v0.4": {
    "n": 393,
    "acc": 0.8219,
    "ece": 0.0292,
    "nll": 0.4943,
    "mean_conf": 0.8179
   },
   "v0.5": {
    "n": 393,
    "acc": 0.8499,
    "ece": 0.0436,
    "nll": 0.4933,
    "mean_conf": 0.8207
   }
  },
  "pr-labels_test-ja": {
   "v0.4": {
    "n": 37,
    "acc": 0.8649,
    "ece": 0.0772,
    "nll": 0.5611,
    "mean_conf": 0.8218
   },
   "v0.5": {
    "n": 37,
    "acc": 0.8649,
    "ece": 0.0775,
    "nll": 0.5395,
    "mean_conf": 0.8212
   }
  },
  "routing_factory-development": {
   "v0.4": {
    "n": 640,
    "acc": 0.9078,
    "ece": 0.073,
    "nll": 0.2972,
    "mean_conf": 0.835
   },
   "v0.5": {
    "n": 640,
    "acc": 0.9047,
    "ece": 0.1301,
    "nll": 0.3357,
    "mean_conf": 0.7806
   }
  },
  "routing_generic-development": {
   "v0.4": {
    "n": 270,
    "acc": 0.9741,
    "ece": 0.0219,
    "nll": 0.074,
    "mean_conf": 0.9631
   },
   "v0.5": {
    "n": 270,
    "acc": 0.9889,
    "ece": 0.0287,
    "nll": 0.0555,
    "mean_conf": 0.9602
   }
  },
  "routing_handlabelled-45": {
   "v0.4": {
    "n": 45,
    "acc": 1.0,
    "ece": 0.0282,
    "nll": 0.0307,
    "mean_conf": 0.9718
   },
   "v0.5": {
    "n": 45,
    "acc": 1.0,
    "ece": 0.0323,
    "nll": 0.0347,
    "mean_conf": 0.9677
   }
  },
  "v4_transfer-v4": {
   "v0.4": {
    "n": 656,
    "acc": 0.689,
    "ece": 0.1276,
    "nll": 0.735,
    "mean_conf": 0.8166
   },
   "v0.5": {
    "n": 656,
    "acc": 0.7165,
    "ece": 0.1059,
    "nll": 0.7034,
    "mean_conf": 0.809
   }
  },
  "v7_decision-v7": {
   "v0.4": {
    "n": 1264,
    "acc": 0.8465,
    "ece": 0.0262,
    "nll": 0.4252,
    "mean_conf": 0.8695
   },
   "v0.5": {
    "n": 1264,
    "acc": 0.8528,
    "ece": 0.0299,
    "nll": 0.4026,
    "mean_conf": 0.882
   }
  }
 },
 "pr_labels_full_bf16": {
  "test": {
   "v0.4": {
    "type": 0.877,
    "sev": 0.778,
    "blast": 0.54,
    "all": 0.8078
   },
   "v0.5": {
    "type": 0.897,
    "sev": 0.756,
    "blast": 0.583,
    "all": 0.8098
   },
   "n_prs": 953
  },
  "test-ja": {
   "v0.4": {
    "all": 0.7409
   },
   "v0.5": {
    "all": 0.7461
   },
   "n_prs": 92
  }
 },
 "owner_prs_hand_checked": {
  "n_prs": 39,
  "v0.4": {
   "type": 0.868,
   "blast": 0.872,
   "sev": 0.656
  },
  "v0.5": {
   "type": 0.842,
   "blast": 0.744,
   "sev": 0.656
  }
 },
 "labeler_replay_review_bot": {
  "n": 98,
  "v0.4": {
   "type": 0.863,
   "blast": 0.704
  },
  "v0.5": {
   "type": 0.863,
   "blast": 0.633
  }
 }
}