| { |
| "base_model": "nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-BF16", |
| "base_revision": "b366b87f762ea334ba3eb55912031baba1515810", |
| "base_weights_sha256": "207483664dfef54c4c4438863260d3ba423bf3d6009b43fe84264b281e9fda23", |
| "release": "state-32k", |
| "selected_step": 8000, |
| "selected_stage_examples": 32000, |
| "precision": "nvfp4", |
| "adapter_sha256": "60723ac470780ac7e8bfe7785c23665be2d5c5d6653cbd2fad653523214aea7d", |
| "weights_sha256": { |
| "model.safetensors": "62bc20a6fbafaf157154650dfb4e9ec32a19d67e8e1f60ddcc3318fd37d6ccb4", |
| "adapter/adapter.safetensors": "60723ac470780ac7e8bfe7785c23665be2d5c5d6653cbd2fad653523214aea7d" |
| }, |
| "evaluated_nvfp4_sha256": "62bc20a6fbafaf157154650dfb4e9ec32a19d67e8e1f60ddcc3318fd37d6ccb4", |
| "accuracy_evaluation_precision": "nvfp4", |
| "bf16_merge": null, |
| "previous_release": { |
| "repo": "rdaum/eider-dory-1.0-4b-decisions-nvfp4", |
| "revision": "8236908a6268050e9cba08f4647d73895be79804", |
| "tag": "actions-48k" |
| }, |
| "training": { |
| "rank": 8, |
| "alpha": 16.0, |
| "loops": 2, |
| "accumulation": 4, |
| "micro_batch_size": 4, |
| "learning_rate": 1e-05, |
| "seed": 20261024, |
| "initialization": { |
| "adapter_sha256": "4b6a812baccf0e4586ab7c31c1574abab979f064ac81dec8baee7e424b659e96", |
| "run_sha256": "5aafa5d840ea5517c42325ed1ce9c61fcf94519b2701c8e9255d6668833f9876", |
| "report_sha256": "39bf8fb9a20c9a811c8907b31bcf93cad312a3869e59c9b0e5e2f4e8f9adbbaf", |
| "parent_completed_steps": 12000, |
| "parent_examples_seen": 48000, |
| "optimizer": "fresh" |
| }, |
| "loss": "soft-target conditional cross-entropy + 0.5 * Brier", |
| "torch": "2.12.0+cu130" |
| }, |
| "training_report": { |
| "adapter_sha256": "60723ac470780ac7e8bfe7785c23665be2d5c5d6653cbd2fad653523214aea7d", |
| "training_state_sha256": "3e39348ed11b584af07de43d1138399d551714226c9703a1c1794853ddb7b6bf", |
| "completed_steps": 8000, |
| "examples_seen": 32000, |
| "training_seconds": 2700.4836061550304, |
| "peak_memory_gib": 52.067845821380615 |
| }, |
| "selection": { |
| "scope": "Original adapter selection / pre-ModelOpt export", |
| "weights_sha256": "9bc834f4dce29a48499b6614c11772d189710138b29c637a384f753b1326b4c0", |
| "report": { |
| "criterion": "NVFP4 validation Brier, cross-entropy, earliest step", |
| "candidates": [ |
| { |
| "step": 0, |
| "examples_seen": 0, |
| "checkpoint": "step-000000", |
| "export": "step-000000", |
| "validation": { |
| "questions": 1024, |
| "modal_accuracy": 0.6005859375, |
| "brier": 0.5027092950491365, |
| "cross_entropy": 1.6357855528374599, |
| "by_kind": { |
| "choice": { |
| "questions": 665, |
| "modal_accuracy": 0.6030075187969924, |
| "brier": 0.4978527031414729, |
| "cross_entropy": 1.564796901472784 |
| }, |
| "noul": { |
| "questions": 144, |
| "modal_accuracy": 0.7361111111111112, |
| "brier": 0.4178836877295325, |
| "cross_entropy": 0.9008511909421006 |
| }, |
| "score": { |
| "questions": 215, |
| "modal_accuracy": 0.5023255813953489, |
| "brier": 0.5745442767822497, |
| "cross_entropy": 2.347590209909279 |
| } |
| }, |
| "by_source": { |
| "alfworld": { |
| "questions": 128, |
| "modal_accuracy": 0.8359375, |
| "brier": 0.19931059120575503, |
| "cross_entropy": 0.7444107758516407 |
| }, |
| "arithmetic": { |
| "questions": 24, |
| "modal_accuracy": 0.7083333333333334, |
| "brier": 0.428241328947602, |
| "cross_entropy": 1.050083253299498 |
| }, |
| "babyai": { |
| "questions": 58, |
| "modal_accuracy": 0.39655172413793105, |
| "brier": 0.7305371209519513, |
| "cross_entropy": 1.4973851876562216 |
| }, |
| "community-diversity-v2/browser_tools/idempotent_retry": { |
| "questions": 2, |
| "modal_accuracy": 1.0, |
| "brier": 4.986938069711992e-05, |
| "cross_entropy": 0.003831617325445817 |
| }, |
| "community-diversity-v2/contracts/governing_law_venue": { |
| "questions": 5, |
| "modal_accuracy": 0.6, |
| "brier": 0.7805844658893524, |
| "cross_entropy": 1.7801497473019683 |
| }, |
| "community-diversity-v2/games/racing_overtake": { |
| "questions": 7, |
| "modal_accuracy": 0.7142857142857143, |
| "brier": 0.4790055889595874, |
| "cross_entropy": 1.1121963454176031 |
| }, |
| "community-diversity-v2/ir/access_schedule": { |
| "questions": 12, |
| "modal_accuracy": 0.6666666666666666, |
| "brier": 0.6101325803838552, |
| "cross_entropy": 2.0033591272403792 |
| }, |
| "community-diversity-v2/rag/policy_exception": { |
| "questions": 10, |
| "modal_accuracy": 0.7, |
| "brier": 0.6003855550573081, |
| "cross_entropy": 1.611779150841159 |
| }, |
| "community-diversity-v2/rubric_judge/trace_consistency": { |
| "questions": 4, |
| "modal_accuracy": 0.75, |
| "brier": 0.42100449664852624, |
| "cross_entropy": 1.9128633478153523 |
| }, |
| "connect_four": { |
| "questions": 56, |
| "modal_accuracy": 0.5357142857142857, |
| "brier": 0.2741592752209492, |
| "cross_entropy": 1.860460665385586 |
| }, |
| "customer-control-v1": { |
| "questions": 7, |
| "modal_accuracy": 1.0, |
| "brier": 2.0002025677443218e-06, |
| "cross_entropy": 0.0004476462766485558 |
| }, |
| "devign": { |
| "questions": 2, |
| "modal_accuracy": 0.5, |
| "brier": 0.41023068518803085, |
| "cross_entropy": 0.6006732852714238 |
| }, |
| "gsm8k": { |
| "questions": 4, |
| "modal_accuracy": 1.0, |
| "brier": 0.0013119361198151522, |
| "cross_entropy": 0.02163855006976333 |
| }, |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.4296875, |
| "brier": 0.5887451086780798, |
| "cross_entropy": 2.801737422463417 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.6041666666666666, |
| "brier": 0.5293956848932907, |
| "cross_entropy": 1.6646787560472525 |
| }, |
| "race": { |
| "questions": 4, |
| "modal_accuracy": 0.5, |
| "brier": 0.8826525379691635, |
| "cross_entropy": 2.2968173663260414 |
| }, |
| "ragtruth": { |
| "questions": 6, |
| "modal_accuracy": 0.6666666666666666, |
| "brier": 0.31120510035647425, |
| "cross_entropy": 0.4402550888479215 |
| }, |
| "scienceworld": { |
| "questions": 128, |
| "modal_accuracy": 0.6953125, |
| "brier": 0.4588827899272394, |
| "cross_entropy": 1.3847298212396886 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.4739583333333333, |
| "brier": 0.8022954836937664, |
| "cross_entropy": 2.0042890400598963 |
| }, |
| "toolace": { |
| "questions": 2, |
| "modal_accuracy": 1.0, |
| "brier": 2.2949296518888497e-08, |
| "cross_entropy": 9.582785411797238e-05 |
| }, |
| "yuri_v3": { |
| "questions": 53, |
| "modal_accuracy": 0.7358490566037735, |
| "brier": 0.0564251534824245, |
| "cross_entropy": 0.8602018822761758 |
| } |
| } |
| } |
| }, |
| { |
| "step": 2000, |
| "examples_seen": 8000, |
| "checkpoint": "step-002000", |
| "export": "step-002000", |
| "validation": { |
| "questions": 1024, |
| "modal_accuracy": 0.7275390625, |
| "brier": 0.31872868889688055, |
| "cross_entropy": 1.116584448395983, |
| "by_kind": { |
| "choice": { |
| "questions": 665, |
| "modal_accuracy": 0.7398496240601504, |
| "brier": 0.3486314543321744, |
| "cross_entropy": 1.1847615210242326 |
| }, |
| "noul": { |
| "questions": 144, |
| "modal_accuracy": 0.8472222222222222, |
| "brier": 0.24375927242920462, |
| "cross_entropy": 0.5036143361404846 |
| }, |
| "score": { |
| "questions": 215, |
| "modal_accuracy": 0.6093023255813953, |
| "brier": 0.27645081427769413, |
| "cross_entropy": 1.3162586012657769 |
| } |
| }, |
| "by_source": { |
| "alfworld": { |
| "questions": 128, |
| "modal_accuracy": 0.8515625, |
| "brier": 0.20112173562767713, |
| "cross_entropy": 0.7568937670273396 |
| }, |
| "arithmetic": { |
| "questions": 24, |
| "modal_accuracy": 0.75, |
| "brier": 0.3778614792863498, |
| "cross_entropy": 1.0070228429929422 |
| }, |
| "babyai": { |
| "questions": 58, |
| "modal_accuracy": 0.5517241379310345, |
| "brier": 0.6998687564446888, |
| "cross_entropy": 1.449144847710849 |
| }, |
| "community-diversity-v2/browser_tools/idempotent_retry": { |
| "questions": 2, |
| "modal_accuracy": 1.0, |
| "brier": 8.502797038864707e-06, |
| "cross_entropy": 0.0017491505823356403 |
| }, |
| "community-diversity-v2/contracts/governing_law_venue": { |
| "questions": 5, |
| "modal_accuracy": 0.6, |
| "brier": 0.64090513735759, |
| "cross_entropy": 1.2842974245060323 |
| }, |
| "community-diversity-v2/games/racing_overtake": { |
| "questions": 7, |
| "modal_accuracy": 0.7142857142857143, |
| "brier": 0.5732229366022509, |
| "cross_entropy": 1.3498112142920367 |
| }, |
| "community-diversity-v2/ir/access_schedule": { |
| "questions": 12, |
| "modal_accuracy": 0.75, |
| "brier": 0.45820962916227576, |
| "cross_entropy": 1.7893452530760972 |
| }, |
| "community-diversity-v2/rag/policy_exception": { |
| "questions": 10, |
| "modal_accuracy": 0.6, |
| "brier": 0.7878584523917267, |
| "cross_entropy": 2.649907510922513 |
| }, |
| "community-diversity-v2/rubric_judge/trace_consistency": { |
| "questions": 4, |
| "modal_accuracy": 0.5, |
| "brier": 0.8062445012416534, |
| "cross_entropy": 1.8181549807503623 |
| }, |
| "connect_four": { |
| "questions": 56, |
| "modal_accuracy": 0.5714285714285714, |
| "brier": 0.268346840698743, |
| "cross_entropy": 1.8491581077937622 |
| }, |
| "customer-control-v1": { |
| "questions": 7, |
| "modal_accuracy": 1.0, |
| "brier": 4.014360934536871e-05, |
| "cross_entropy": 0.0018805145026462434 |
| }, |
| "devign": { |
| "questions": 2, |
| "modal_accuracy": 1.0, |
| "brier": 0.43785613217853164, |
| "cross_entropy": 0.6306845393838758 |
| }, |
| "gsm8k": { |
| "questions": 4, |
| "modal_accuracy": 1.0, |
| "brier": 0.0023963047501729595, |
| "cross_entropy": 0.02626187987425209 |
| }, |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.546875, |
| "brier": 0.20973734203719346, |
| "cross_entropy": 1.5167245986871056 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.9114583333333334, |
| "brier": 0.1816467064252445, |
| "cross_entropy": 0.8230249970733392 |
| }, |
| "race": { |
| "questions": 4, |
| "modal_accuracy": 0.5, |
| "brier": 0.7315860423730358, |
| "cross_entropy": 2.0558892146843153 |
| }, |
| "ragtruth": { |
| "questions": 6, |
| "modal_accuracy": 0.5, |
| "brier": 0.6525781536661275, |
| "cross_entropy": 0.9808095278526777 |
| }, |
| "scienceworld": { |
| "questions": 128, |
| "modal_accuracy": 0.6953125, |
| "brier": 0.430457617431226, |
| "cross_entropy": 1.3267025517668618 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.6979166666666666, |
| "brier": 0.44132333714721916, |
| "cross_entropy": 0.939129528604241 |
| }, |
| "toolace": { |
| "questions": 2, |
| "modal_accuracy": 1.0, |
| "brier": 1.4302336637850583e-07, |
| "cross_entropy": 0.00026036870363595557 |
| }, |
| "yuri_v3": { |
| "questions": 53, |
| "modal_accuracy": 0.7358490566037735, |
| "brier": 0.05366664986435064, |
| "cross_entropy": 0.8647152527990684 |
| } |
| } |
| } |
| }, |
| { |
| "step": 4000, |
| "examples_seen": 16000, |
| "checkpoint": "step-004000", |
| "export": "step-004000", |
| "validation": { |
| "questions": 1024, |
| "modal_accuracy": 0.740234375, |
| "brier": 0.31259603426070215, |
| "cross_entropy": 1.0641685621781927, |
| "by_kind": { |
| "choice": { |
| "questions": 665, |
| "modal_accuracy": 0.7413533834586467, |
| "brier": 0.35520899641938364, |
| "cross_entropy": 1.150404749554445 |
| }, |
| "noul": { |
| "questions": 144, |
| "modal_accuracy": 0.8333333333333334, |
| "brier": 0.2587362169867197, |
| "cross_entropy": 0.527162702406588 |
| }, |
| "score": { |
| "questions": 215, |
| "modal_accuracy": 0.6744186046511628, |
| "brier": 0.2168667033394475, |
| "cross_entropy": 1.157107070094022 |
| } |
| }, |
| "by_source": { |
| "alfworld": { |
| "questions": 128, |
| "modal_accuracy": 0.859375, |
| "brier": 0.1875144540034567, |
| "cross_entropy": 0.6708904667559872 |
| }, |
| "arithmetic": { |
| "questions": 24, |
| "modal_accuracy": 0.7083333333333334, |
| "brier": 0.4071248120462074, |
| "cross_entropy": 1.072867837361791 |
| }, |
| "babyai": { |
| "questions": 58, |
| "modal_accuracy": 0.4827586206896552, |
| "brier": 0.7226287558174715, |
| "cross_entropy": 1.4270151869584489 |
| }, |
| "community-diversity-v2/browser_tools/idempotent_retry": { |
| "questions": 2, |
| "modal_accuracy": 1.0, |
| "brier": 6.599285085529234e-06, |
| "cross_entropy": 0.0016185379703032327 |
| }, |
| "community-diversity-v2/contracts/governing_law_venue": { |
| "questions": 5, |
| "modal_accuracy": 0.8, |
| "brier": 0.39998067070790544, |
| "cross_entropy": 1.08609476980073 |
| }, |
| "community-diversity-v2/games/racing_overtake": { |
| "questions": 7, |
| "modal_accuracy": 0.8571428571428571, |
| "brier": 0.28687468365522234, |
| "cross_entropy": 0.7248149597143485 |
| }, |
| "community-diversity-v2/ir/access_schedule": { |
| "questions": 12, |
| "modal_accuracy": 0.75, |
| "brier": 0.477438586205062, |
| "cross_entropy": 1.9109389754190291 |
| }, |
| "community-diversity-v2/rag/policy_exception": { |
| "questions": 10, |
| "modal_accuracy": 0.6, |
| "brier": 0.7693115419877218, |
| "cross_entropy": 2.051549980800976 |
| }, |
| "community-diversity-v2/rubric_judge/trace_consistency": { |
| "questions": 4, |
| "modal_accuracy": 0.25, |
| "brier": 1.1149087692116901, |
| "cross_entropy": 2.466424832470031 |
| }, |
| "connect_four": { |
| "questions": 56, |
| "modal_accuracy": 0.6428571428571429, |
| "brier": 0.26805081416207754, |
| "cross_entropy": 1.8479820614570504 |
| }, |
| "customer-control-v1": { |
| "questions": 7, |
| "modal_accuracy": 1.0, |
| "brier": 1.6518035601985627e-05, |
| "cross_entropy": 0.0012598198888345246 |
| }, |
| "devign": { |
| "questions": 2, |
| "modal_accuracy": 0.5, |
| "brier": 0.43338634881169164, |
| "cross_entropy": 0.6242116836662074 |
| }, |
| "gsm8k": { |
| "questions": 4, |
| "modal_accuracy": 1.0, |
| "brier": 0.0005199031989910008, |
| "cross_entropy": 0.013970904245707016 |
| }, |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.5625, |
| "brier": 0.2123470190045601, |
| "cross_entropy": 1.5161004021555795 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.9010416666666666, |
| "brier": 0.18652079389211781, |
| "cross_entropy": 0.7723674627217165 |
| }, |
| "race": { |
| "questions": 4, |
| "modal_accuracy": 0.5, |
| "brier": 0.6806357215652038, |
| "cross_entropy": 1.9339496668179939 |
| }, |
| "ragtruth": { |
| "questions": 6, |
| "modal_accuracy": 0.6666666666666666, |
| "brier": 0.5780328053534197, |
| "cross_entropy": 0.9064917649110132 |
| }, |
| "scienceworld": { |
| "questions": 128, |
| "modal_accuracy": 0.6796875, |
| "brier": 0.49342345206712573, |
| "cross_entropy": 1.4302781643246851 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.765625, |
| "brier": 0.3713316872108095, |
| "cross_entropy": 0.7410961018920553 |
| }, |
| "toolace": { |
| "questions": 2, |
| "modal_accuracy": 1.0, |
| "brier": 3.664008183371336e-07, |
| "cross_entropy": 0.00041208643515311584 |
| }, |
| "yuri_v3": { |
| "questions": 53, |
| "modal_accuracy": 0.7547169811320755, |
| "brier": 0.056719285382028715, |
| "cross_entropy": 0.8641864337643139 |
| } |
| } |
| } |
| }, |
| { |
| "step": 8000, |
| "examples_seen": 32000, |
| "checkpoint": "step-008000", |
| "export": "step-008000", |
| "validation": { |
| "questions": 1024, |
| "modal_accuracy": 0.76171875, |
| "brier": 0.27786017825450965, |
| "cross_entropy": 0.9927829125866763, |
| "by_kind": { |
| "choice": { |
| "questions": 665, |
| "modal_accuracy": 0.7654135338345864, |
| "brier": 0.3189465668614991, |
| "cross_entropy": 1.0534065611001757 |
| }, |
| "noul": { |
| "questions": 144, |
| "modal_accuracy": 0.8958333333333334, |
| "brier": 0.16471210037099535, |
| "cross_entropy": 0.39482479972079665 |
| }, |
| "score": { |
| "questions": 215, |
| "modal_accuracy": 0.6604651162790698, |
| "brier": 0.22656192147115203, |
| "cross_entropy": 1.2057654334760237 |
| } |
| }, |
| "by_source": { |
| "alfworld": { |
| "questions": 128, |
| "modal_accuracy": 0.8671875, |
| "brier": 0.1790825849971539, |
| "cross_entropy": 0.6080971490404414 |
| }, |
| "arithmetic": { |
| "questions": 24, |
| "modal_accuracy": 0.6666666666666666, |
| "brier": 0.5263100827359132, |
| "cross_entropy": 1.403012675529207 |
| }, |
| "babyai": { |
| "questions": 58, |
| "modal_accuracy": 0.5517241379310345, |
| "brier": 0.684833327087956, |
| "cross_entropy": 1.5818612127101548 |
| }, |
| "community-diversity-v2/browser_tools/idempotent_retry": { |
| "questions": 2, |
| "modal_accuracy": 1.0, |
| "brier": 8.422208826818114e-06, |
| "cross_entropy": 0.0016334122276107624 |
| }, |
| "community-diversity-v2/contracts/governing_law_venue": { |
| "questions": 5, |
| "modal_accuracy": 0.8, |
| "brier": 0.39072800981047207, |
| "cross_entropy": 0.8926558622999824 |
| }, |
| "community-diversity-v2/games/racing_overtake": { |
| "questions": 7, |
| "modal_accuracy": 0.8571428571428571, |
| "brier": 0.28439123582360026, |
| "cross_entropy": 0.7928804085918381 |
| }, |
| "community-diversity-v2/ir/access_schedule": { |
| "questions": 12, |
| "modal_accuracy": 0.6666666666666666, |
| "brier": 0.6286826358235, |
| "cross_entropy": 2.0004531092707 |
| }, |
| "community-diversity-v2/rag/policy_exception": { |
| "questions": 10, |
| "modal_accuracy": 0.7, |
| "brier": 0.44538602593083176, |
| "cross_entropy": 1.117689014223367 |
| }, |
| "community-diversity-v2/rubric_judge/trace_consistency": { |
| "questions": 4, |
| "modal_accuracy": 0.25, |
| "brier": 1.4342464951649092, |
| "cross_entropy": 3.381933827755728 |
| }, |
| "connect_four": { |
| "questions": 56, |
| "modal_accuracy": 0.5535714285714286, |
| "brier": 0.2622696208064482, |
| "cross_entropy": 1.8219529101323455 |
| }, |
| "customer-control-v1": { |
| "questions": 7, |
| "modal_accuracy": 1.0, |
| "brier": 2.417104630113183e-06, |
| "cross_entropy": 0.0005900334430327367 |
| }, |
| "devign": { |
| "questions": 2, |
| "modal_accuracy": 1.0, |
| "brier": 0.43780762834608883, |
| "cross_entropy": 0.6306076256145883 |
| }, |
| "gsm8k": { |
| "questions": 4, |
| "modal_accuracy": 1.0, |
| "brier": 0.0003744313065983422, |
| "cross_entropy": 0.011988960485765076 |
| }, |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.5859375, |
| "brier": 0.20697473621325052, |
| "cross_entropy": 1.5034032587243362 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.9635416666666666, |
| "brier": 0.11802009440200961, |
| "cross_entropy": 0.6579051066101222 |
| }, |
| "race": { |
| "questions": 4, |
| "modal_accuracy": 0.5, |
| "brier": 0.7315329429054658, |
| "cross_entropy": 2.0647249708287245 |
| }, |
| "ragtruth": { |
| "questions": 6, |
| "modal_accuracy": 0.6666666666666666, |
| "brier": 0.609860991064251, |
| "cross_entropy": 0.9741778978250341 |
| }, |
| "scienceworld": { |
| "questions": 128, |
| "modal_accuracy": 0.703125, |
| "brier": 0.44470820194957994, |
| "cross_entropy": 1.2102900099661185 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.796875, |
| "brier": 0.29338767561802165, |
| "cross_entropy": 0.6140028830315581 |
| }, |
| "toolace": { |
| "questions": 2, |
| "modal_accuracy": 1.0, |
| "brier": 2.1804417353162663e-07, |
| "cross_entropy": 0.0003234946827488933 |
| }, |
| "yuri_v3": { |
| "questions": 53, |
| "modal_accuracy": 0.7169811320754716, |
| "brier": 0.056977519481840835, |
| "cross_entropy": 0.860752644211149 |
| } |
| } |
| } |
| } |
| ], |
| "winner": { |
| "step": 8000, |
| "examples_seen": 32000, |
| "checkpoint": "step-008000", |
| "export": "step-008000", |
| "validation": { |
| "questions": 1024, |
| "modal_accuracy": 0.76171875, |
| "brier": 0.27786017825450965, |
| "cross_entropy": 0.9927829125866763, |
| "by_kind": { |
| "choice": { |
| "questions": 665, |
| "modal_accuracy": 0.7654135338345864, |
| "brier": 0.3189465668614991, |
| "cross_entropy": 1.0534065611001757 |
| }, |
| "noul": { |
| "questions": 144, |
| "modal_accuracy": 0.8958333333333334, |
| "brier": 0.16471210037099535, |
| "cross_entropy": 0.39482479972079665 |
| }, |
| "score": { |
| "questions": 215, |
| "modal_accuracy": 0.6604651162790698, |
| "brier": 0.22656192147115203, |
| "cross_entropy": 1.2057654334760237 |
| } |
| }, |
| "by_source": { |
| "alfworld": { |
| "questions": 128, |
| "modal_accuracy": 0.8671875, |
| "brier": 0.1790825849971539, |
| "cross_entropy": 0.6080971490404414 |
| }, |
| "arithmetic": { |
| "questions": 24, |
| "modal_accuracy": 0.6666666666666666, |
| "brier": 0.5263100827359132, |
| "cross_entropy": 1.403012675529207 |
| }, |
| "babyai": { |
| "questions": 58, |
| "modal_accuracy": 0.5517241379310345, |
| "brier": 0.684833327087956, |
| "cross_entropy": 1.5818612127101548 |
| }, |
| "community-diversity-v2/browser_tools/idempotent_retry": { |
| "questions": 2, |
| "modal_accuracy": 1.0, |
| "brier": 8.422208826818114e-06, |
| "cross_entropy": 0.0016334122276107624 |
| }, |
| "community-diversity-v2/contracts/governing_law_venue": { |
| "questions": 5, |
| "modal_accuracy": 0.8, |
| "brier": 0.39072800981047207, |
| "cross_entropy": 0.8926558622999824 |
| }, |
| "community-diversity-v2/games/racing_overtake": { |
| "questions": 7, |
| "modal_accuracy": 0.8571428571428571, |
| "brier": 0.28439123582360026, |
| "cross_entropy": 0.7928804085918381 |
| }, |
| "community-diversity-v2/ir/access_schedule": { |
| "questions": 12, |
| "modal_accuracy": 0.6666666666666666, |
| "brier": 0.6286826358235, |
| "cross_entropy": 2.0004531092707 |
| }, |
| "community-diversity-v2/rag/policy_exception": { |
| "questions": 10, |
| "modal_accuracy": 0.7, |
| "brier": 0.44538602593083176, |
| "cross_entropy": 1.117689014223367 |
| }, |
| "community-diversity-v2/rubric_judge/trace_consistency": { |
| "questions": 4, |
| "modal_accuracy": 0.25, |
| "brier": 1.4342464951649092, |
| "cross_entropy": 3.381933827755728 |
| }, |
| "connect_four": { |
| "questions": 56, |
| "modal_accuracy": 0.5535714285714286, |
| "brier": 0.2622696208064482, |
| "cross_entropy": 1.8219529101323455 |
| }, |
| "customer-control-v1": { |
| "questions": 7, |
| "modal_accuracy": 1.0, |
| "brier": 2.417104630113183e-06, |
| "cross_entropy": 0.0005900334430327367 |
| }, |
| "devign": { |
| "questions": 2, |
| "modal_accuracy": 1.0, |
| "brier": 0.43780762834608883, |
| "cross_entropy": 0.6306076256145883 |
| }, |
| "gsm8k": { |
| "questions": 4, |
| "modal_accuracy": 1.0, |
| "brier": 0.0003744313065983422, |
| "cross_entropy": 0.011988960485765076 |
| }, |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.5859375, |
| "brier": 0.20697473621325052, |
| "cross_entropy": 1.5034032587243362 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.9635416666666666, |
| "brier": 0.11802009440200961, |
| "cross_entropy": 0.6579051066101222 |
| }, |
| "race": { |
| "questions": 4, |
| "modal_accuracy": 0.5, |
| "brier": 0.7315329429054658, |
| "cross_entropy": 2.0647249708287245 |
| }, |
| "ragtruth": { |
| "questions": 6, |
| "modal_accuracy": 0.6666666666666666, |
| "brier": 0.609860991064251, |
| "cross_entropy": 0.9741778978250341 |
| }, |
| "scienceworld": { |
| "questions": 128, |
| "modal_accuracy": 0.703125, |
| "brier": 0.44470820194957994, |
| "cross_entropy": 1.2102900099661185 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.796875, |
| "brier": 0.29338767561802165, |
| "cross_entropy": 0.6140028830315581 |
| }, |
| "toolace": { |
| "questions": 2, |
| "modal_accuracy": 1.0, |
| "brier": 2.1804417353162663e-07, |
| "cross_entropy": 0.0003234946827488933 |
| }, |
| "yuri_v3": { |
| "questions": 53, |
| "modal_accuracy": 0.7169811320754716, |
| "brier": 0.056977519481840835, |
| "cross_entropy": 0.860752644211149 |
| } |
| } |
| } |
| }, |
| "validation_split": "new_validation" |
| } |
| }, |
| "calibration": { |
| "status": "uncalibrated answer probabilities", |
| "included": false, |
| "enabled_by_default": false |
| }, |
| "quantization": { |
| "producer": { |
| "name": "modelopt", |
| "version": "0.0.1.dev1+gfcc59e3a9" |
| }, |
| "algorithm": "local_hessian", |
| "calibration_examples": 256, |
| "group_size": 16, |
| "activation_precision": "bf16", |
| "source_bf16_sha256": { |
| "model-00001-of-00008.safetensors": "f8ae89948adc35f42b2d735abc460bf20df89fe27063009e7023af236421943e", |
| "model-00002-of-00008.safetensors": "09c16bdfe0f5fb160048a35f08ca998fa1e704efdcb4b3d504e46b2870c68e2d", |
| "model-00003-of-00008.safetensors": "978f4d0f66dce924a81ee2e57aaaa4d0431bf5a163eb3445f32414637846b74c", |
| "model-00004-of-00008.safetensors": "056ce5b367b2d955ce08ec5795d12ba6475640df7175c9f3eb7d5d35e18ddd86", |
| "model-00005-of-00008.safetensors": "dbc45d99c137991456115cd9b71e8fe69dcb7d5b923cee93835e34646af9da0d", |
| "model-00006-of-00008.safetensors": "bd559f8be05597b9915e91c8bd449d7c5194a2205c5a02e4ec750971a3f2814e", |
| "model-00007-of-00008.safetensors": "6c1ebd7492594617d7e0be46cce8b05d57f27930dac431bff1c23a3d496782a3", |
| "model-00008-of-00008.safetensors": "09dde4e45f4cc217f00222f3c57d8b8656d12a8f215e09d300a6f98d434fe80e" |
| }, |
| "export_metadata": "export.json" |
| }, |
| "published_nvfp4_sha256": "62bc20a6fbafaf157154650dfb4e9ec32a19d67e8e1f60ddcc3318fd37d6ccb4", |
| "evaluation": { |
| "test": "../evaluation/base-comparison.json", |
| "validation": "../evaluation/modelopt-validation.json", |
| "jevman": "../evaluation/jevman-comparison.json" |
| } |
| } |
|
|
