| { |
| "test": { |
| "schema": "dory-base-vs-typed-decisions/v1", |
| "base_model": "nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-BF16", |
| "base_model_revision": "b366b87f762ea334ba3eb55912031baba1515810", |
| "format": "NVFP4", |
| "quantization_recipe": "NVIDIA Model Optimizer local_hessian", |
| "calibration_examples": 256, |
| "loops": 2, |
| "calibrated": false, |
| "prompt": "Identical typed-decision prompts and allowed-answer scoring; no generated reasoning.", |
| "binary_sha256": "935335c9df451b4b9108d65fc46882abbfd7a0b6cb20b3ba76f4e8c42fa7ab0c", |
| "scoring_script_sha256": "1a8acf16b2fd060500689ad2580e3c0709ec07647265067fe57e168764b56b42", |
| "test_data_sha256": "e2ca7413a306fc2b0e9e044aa685c94d5aa8e0def7cfcec6448941b7369ae625", |
| "base_weight_sha256": "2dc8446f2c52d3249eac5bd2154f1f4a8f206152e3b66d7077965b9ad991bbf7", |
| "base": { |
| "questions": 512, |
| "modal_accuracy": 0.376953125, |
| "brier": 0.6512325676171807, |
| "cross_entropy": 1.9877935454851494, |
| "mean_request_ms": 81.26816220556066, |
| "median_request_ms": 72.5804881658405, |
| "by_kind": { |
| "choice": { |
| "questions": 241, |
| "modal_accuracy": 0.2863070539419087, |
| "brier": 0.742723819745487, |
| "cross_entropy": 2.2877653477997058, |
| "mean_request_ms": 75.63938830179235, |
| "median_request_ms": 72.81611207872629 |
| }, |
| "noul": { |
| "questions": 94, |
| "modal_accuracy": 0.5, |
| "brier": 0.670176294758546, |
| "cross_entropy": 0.9175096541022592, |
| "mean_request_ms": 68.15949506423574, |
| "median_request_ms": 68.27253755182028 |
| }, |
| "score": { |
| "questions": 177, |
| "modal_accuracy": 0.4350282485875706, |
| "brier": 0.5165992223391576, |
| "cross_entropy": 2.1477567174183902, |
| "mean_request_ms": 95.89386402529344, |
| "median_request_ms": 77.44529517367482 |
| } |
| }, |
| "by_source": { |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.484375, |
| "brier": 0.31624743248922543, |
| "cross_entropy": 1.797596837336767, |
| "mean_request_ms": 107.6381878774555, |
| "median_request_ms": 96.75377863459289 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.3697916666666667, |
| "brier": 0.6013039965397795, |
| "cross_entropy": 1.8274480710869951, |
| "mean_request_ms": 75.0043233759546, |
| "median_request_ms": 73.60281050205231 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.3125, |
| "brier": 0.9244845621132188, |
| "cross_entropy": 2.274936825315559, |
| "mean_request_ms": 69.95198392057016, |
| "median_request_ms": 69.4849812425673 |
| } |
| } |
| }, |
| "fine_tuned_weight_sha256": "62bc20a6fbafaf157154650dfb4e9ec32a19d67e8e1f60ddcc3318fd37d6ccb4", |
| "fine_tuned": { |
| "questions": 512, |
| "modal_accuracy": 0.771484375, |
| "brier": 0.26463106057883456, |
| "cross_entropy": 1.075109941121327, |
| "mean_request_ms": 80.6613628164996, |
| "median_request_ms": 71.86161540448666, |
| "by_kind": { |
| "choice": { |
| "questions": 241, |
| "modal_accuracy": 0.8215767634854771, |
| "brier": 0.34044989428658823, |
| "cross_entropy": 1.1801933058267577, |
| "mean_request_ms": 75.06443578865827, |
| "median_request_ms": 72.3452023230493 |
| }, |
| "noul": { |
| "questions": 94, |
| "modal_accuracy": 0.9574468085106383, |
| "brier": 0.09152335422286362, |
| "cross_entropy": 0.2881069286288201, |
| "mean_request_ms": 67.15274397581022, |
| "median_request_ms": 67.46977800503373 |
| }, |
| "score": { |
| "questions": 177, |
| "modal_accuracy": 0.6045197740112994, |
| "brier": 0.2533304135386799, |
| "cross_entropy": 1.3499867336653204, |
| "mean_request_ms": 95.45610623307905, |
| "median_request_ms": 77.96838786453009 |
| } |
| }, |
| "by_source": { |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.5078125, |
| "brier": 0.22775570941129228, |
| "cross_entropy": 1.5323848707632575, |
| "mean_request_ms": 107.58943439577706, |
| "median_request_ms": 97.03774400986731 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.9635416666666666, |
| "brier": 0.155754142490034, |
| "cross_entropy": 0.834348251030471, |
| "mean_request_ms": 74.49704928634067, |
| "median_request_ms": 73.17394809797406 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.7552083333333334, |
| "brier": 0.3980915461126633, |
| "cross_entropy": 1.0110216781175627, |
| "mean_request_ms": 68.87362862714023, |
| "median_request_ms": 68.2264359202236 |
| } |
| } |
| } |
| }, |
| "validation": { |
| "questions": 1024, |
| "split": "validation", |
| "selection_use": "Quantization recipe comparison; not an independent test", |
| "loops": 2, |
| "probability_calibration": false, |
| "calibration_examples": 256, |
| "calibration_source": "Training examples disjoint from validation by ID and source group", |
| "weights_sha256": "62bc20a6fbafaf157154650dfb4e9ec32a19d67e8e1f60ddcc3318fd37d6ccb4", |
| "metrics": { |
| "questions": 1024, |
| "correct": 0.78515625, |
| "brier": 0.256938595885194, |
| "cross_entropy": 0.9667092188545374 |
| }, |
| "by_kind": { |
| "choice": { |
| "questions": 665, |
| "correct": 0.7819548872180451, |
| "brier": 0.29931089665593197, |
| "cross_entropy": 1.021091694949712 |
| }, |
| "noul": { |
| "questions": 144, |
| "correct": 0.9097222222222222, |
| "brier": 0.146691427234091, |
| "cross_entropy": 0.4264544071293922 |
| }, |
| "score": { |
| "questions": 215, |
| "correct": 0.7116279069767442, |
| "brier": 0.19972004831876652, |
| "cross_entropy": 1.160348038785374 |
| } |
| } |
| }, |
| "scope": "Test questions are disjoint from calibration training examples. Validation was used for quantization selection." |
| } |
|
|
