{
  "test": {
    "schema": "dory-base-vs-typed-decisions/v1",
    "base_model": "nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-BF16",
    "base_model_revision": "b366b87f762ea334ba3eb55912031baba1515810",
    "format": "NVFP4",
    "quantization_recipe": "NVIDIA Model Optimizer local_hessian",
    "calibration_examples": 256,
    "loops": 2,
    "calibrated": false,
    "prompt": "Identical typed-decision prompts and allowed-answer scoring; no generated reasoning.",
    "binary_sha256": "935335c9df451b4b9108d65fc46882abbfd7a0b6cb20b3ba76f4e8c42fa7ab0c",
    "scoring_script_sha256": "1a8acf16b2fd060500689ad2580e3c0709ec07647265067fe57e168764b56b42",
    "test_data_sha256": "e2ca7413a306fc2b0e9e044aa685c94d5aa8e0def7cfcec6448941b7369ae625",
    "base_weight_sha256": "2dc8446f2c52d3249eac5bd2154f1f4a8f206152e3b66d7077965b9ad991bbf7",
    "base": {
      "questions": 512,
      "modal_accuracy": 0.376953125,
      "brier": 0.6512325676171807,
      "cross_entropy": 1.9877935454851494,
      "mean_request_ms": 81.26816220556066,
      "median_request_ms": 72.5804881658405,
      "by_kind": {
        "choice": {
          "questions": 241,
          "modal_accuracy": 0.2863070539419087,
          "brier": 0.742723819745487,
          "cross_entropy": 2.2877653477997058,
          "mean_request_ms": 75.63938830179235,
          "median_request_ms": 72.81611207872629
        },
        "noul": {
          "questions": 94,
          "modal_accuracy": 0.5,
          "brier": 0.670176294758546,
          "cross_entropy": 0.9175096541022592,
          "mean_request_ms": 68.15949506423574,
          "median_request_ms": 68.27253755182028
        },
        "score": {
          "questions": 177,
          "modal_accuracy": 0.4350282485875706,
          "brier": 0.5165992223391576,
          "cross_entropy": 2.1477567174183902,
          "mean_request_ms": 95.89386402529344,
          "median_request_ms": 77.44529517367482
        }
      },
      "by_source": {
        "helpsteer2": {
          "questions": 128,
          "modal_accuracy": 0.484375,
          "brier": 0.31624743248922543,
          "cross_entropy": 1.797596837336767,
          "mean_request_ms": 107.6381878774555,
          "median_request_ms": 96.75377863459289
        },
        "planning": {
          "questions": 192,
          "modal_accuracy": 0.3697916666666667,
          "brier": 0.6013039965397795,
          "cross_entropy": 1.8274480710869951,
          "mean_request_ms": 75.0043233759546,
          "median_request_ms": 73.60281050205231
        },
        "scone": {
          "questions": 192,
          "modal_accuracy": 0.3125,
          "brier": 0.9244845621132188,
          "cross_entropy": 2.274936825315559,
          "mean_request_ms": 69.95198392057016,
          "median_request_ms": 69.4849812425673
        }
      }
    },
    "fine_tuned_weight_sha256": "62bc20a6fbafaf157154650dfb4e9ec32a19d67e8e1f60ddcc3318fd37d6ccb4",
    "fine_tuned": {
      "questions": 512,
      "modal_accuracy": 0.771484375,
      "brier": 0.26463106057883456,
      "cross_entropy": 1.075109941121327,
      "mean_request_ms": 80.6613628164996,
      "median_request_ms": 71.86161540448666,
      "by_kind": {
        "choice": {
          "questions": 241,
          "modal_accuracy": 0.8215767634854771,
          "brier": 0.34044989428658823,
          "cross_entropy": 1.1801933058267577,
          "mean_request_ms": 75.06443578865827,
          "median_request_ms": 72.3452023230493
        },
        "noul": {
          "questions": 94,
          "modal_accuracy": 0.9574468085106383,
          "brier": 0.09152335422286362,
          "cross_entropy": 0.2881069286288201,
          "mean_request_ms": 67.15274397581022,
          "median_request_ms": 67.46977800503373
        },
        "score": {
          "questions": 177,
          "modal_accuracy": 0.6045197740112994,
          "brier": 0.2533304135386799,
          "cross_entropy": 1.3499867336653204,
          "mean_request_ms": 95.45610623307905,
          "median_request_ms": 77.96838786453009
        }
      },
      "by_source": {
        "helpsteer2": {
          "questions": 128,
          "modal_accuracy": 0.5078125,
          "brier": 0.22775570941129228,
          "cross_entropy": 1.5323848707632575,
          "mean_request_ms": 107.58943439577706,
          "median_request_ms": 97.03774400986731
        },
        "planning": {
          "questions": 192,
          "modal_accuracy": 0.9635416666666666,
          "brier": 0.155754142490034,
          "cross_entropy": 0.834348251030471,
          "mean_request_ms": 74.49704928634067,
          "median_request_ms": 73.17394809797406
        },
        "scone": {
          "questions": 192,
          "modal_accuracy": 0.7552083333333334,
          "brier": 0.3980915461126633,
          "cross_entropy": 1.0110216781175627,
          "mean_request_ms": 68.87362862714023,
          "median_request_ms": 68.2264359202236
        }
      }
    }
  },
  "validation": {
    "questions": 1024,
    "split": "validation",
    "selection_use": "Quantization recipe comparison; not an independent test",
    "loops": 2,
    "probability_calibration": false,
    "calibration_examples": 256,
    "calibration_source": "Training examples disjoint from validation by ID and source group",
    "weights_sha256": "62bc20a6fbafaf157154650dfb4e9ec32a19d67e8e1f60ddcc3318fd37d6ccb4",
    "metrics": {
      "questions": 1024,
      "correct": 0.78515625,
      "brier": 0.256938595885194,
      "cross_entropy": 0.9667092188545374
    },
    "by_kind": {
      "choice": {
        "questions": 665,
        "correct": 0.7819548872180451,
        "brier": 0.29931089665593197,
        "cross_entropy": 1.021091694949712
      },
      "noul": {
        "questions": 144,
        "correct": 0.9097222222222222,
        "brier": 0.146691427234091,
        "cross_entropy": 0.4264544071293922
      },
      "score": {
        "questions": 215,
        "correct": 0.7116279069767442,
        "brier": 0.19972004831876652,
        "cross_entropy": 1.160348038785374
      }
    }
  },
  "scope": "Test questions are disjoint from calibration training examples. Validation was used for quantization selection."
}