| { |
| "scope": "Exploratory: these test splits have previously informed development.", |
| "seeds": [ |
| 17, |
| 42, |
| 83 |
| ], |
| "torch": "2.11.0+cu128", |
| "device": "cuda:0", |
| "arms": { |
| "reinforce": { |
| "uncalibrated": { |
| "overall_brier": { |
| "mean": 0.10294622417491706, |
| "sample_std": 0.002599080705236424, |
| "values": [ |
| 0.10353795868889026, |
| 0.10519841661327942, |
| 0.10010229722258146 |
| ] |
| }, |
| "overall_accuracy": { |
| "mean": 0.9291666666666667, |
| "sample_std": 0.0036084391824351483, |
| "values": [ |
| 0.93125, |
| 0.925, |
| 0.93125 |
| ] |
| }, |
| "overall_nll": { |
| "mean": 0.1551456288790227, |
| "sample_std": 0.0036213881972196487, |
| "values": [ |
| 0.15628733976823508, |
| 0.15805856801203613, |
| 0.15109097885679681 |
| ] |
| }, |
| "escalation_brier": { |
| "mean": 0.39582496485276236, |
| "sample_std": 0.015027460780644108, |
| "values": [ |
| 0.3894728404020668, |
| 0.4129853920524647, |
| 0.3850166621037557 |
| ] |
| }, |
| "escalation_oracle_error": { |
| "mean": 0.029907245868575776, |
| "sample_std": 0.0054552554318909674, |
| "values": [ |
| 0.028814277704596627, |
| 0.03582624121361282, |
| 0.02508121868751787 |
| ] |
| }, |
| "department_brier": { |
| "mean": 0.015501622589877693, |
| "sample_std": 0.008755121273280025, |
| "values": [ |
| 0.02446606011017072, |
| 0.006972022766819956, |
| 0.015066784892642397 |
| ] |
| }, |
| "department_accuracy": { |
| "mean": 0.9883333333333333, |
| "sample_std": 0.00763762615825974, |
| "values": [ |
| 0.98, |
| 0.995, |
| 0.99 |
| ] |
| } |
| }, |
| "global": { |
| "overall_brier": { |
| "mean": 0.10175951933231486, |
| "sample_std": 0.0019328278968226429, |
| "values": [ |
| 0.1029542671867347, |
| 0.10279470684407294, |
| 0.09952958396613695 |
| ] |
| }, |
| "overall_accuracy": { |
| "mean": 0.9291666666666667, |
| "sample_std": 0.0036084391824351483, |
| "values": [ |
| 0.93125, |
| 0.925, |
| 0.93125 |
| ] |
| }, |
| "overall_nll": { |
| "mean": 0.15411299632607459, |
| "sample_std": 0.003556021658733511, |
| "values": [ |
| 0.15745305743150984, |
| 0.15451136706332724, |
| 0.15037456448338665 |
| ] |
| }, |
| "escalation_brier": { |
| "mean": 0.3900226244591283, |
| "sample_std": 0.010218525967785686, |
| "values": [ |
| 0.38630746982104214, |
| 0.4015789899739018, |
| 0.38218141358244084 |
| ] |
| }, |
| "escalation_oracle_error": { |
| "mean": 0.032096277186036824, |
| "sample_std": 0.003198074435346083, |
| "values": [ |
| 0.03361730003959572, |
| 0.03424996328228628, |
| 0.02842156823622846 |
| ] |
| }, |
| "department_brier": { |
| "mean": 0.016163031713855146, |
| "sample_std": 0.008499735845706667, |
| "values": [ |
| 0.0250169991241882, |
| 0.008068493609724506, |
| 0.01540360240765274 |
| ] |
| }, |
| "department_accuracy": { |
| "mean": 0.9883333333333333, |
| "sample_std": 0.00763762615825974, |
| "values": [ |
| 0.98, |
| 0.995, |
| 0.99 |
| ] |
| } |
| }, |
| "global_temperature": { |
| "mean": 1.240038419401916, |
| "sample_std": 0.03271323042554636, |
| "values": [ |
| 1.2589254117941675, |
| 1.2589254117941675, |
| 1.2022644346174132 |
| ] |
| } |
| }, |
| "pathwise": { |
| "uncalibrated": { |
| "overall_brier": { |
| "mean": 0.0994051174708289, |
| "sample_std": 0.000828672884992785, |
| "values": [ |
| 0.09962411375196488, |
| 0.09848894143794036, |
| 0.10010229722258146 |
| ] |
| }, |
| "overall_accuracy": { |
| "mean": 0.93375, |
| "sample_std": 0.0033071891388307306, |
| "values": [ |
| 0.9375, |
| 0.9325, |
| 0.93125 |
| ] |
| }, |
| "overall_nll": { |
| "mean": 0.1493629731751893, |
| "sample_std": 0.002261727963142961, |
| "values": [ |
| 0.15019482134568096, |
| 0.14680311932309004, |
| 0.15109097885679681 |
| ] |
| }, |
| "escalation_brier": { |
| "mean": 0.39087688054770786, |
| "sample_std": 0.005834194495826293, |
| "values": [ |
| 0.3966846981039322, |
| 0.39092928143543576, |
| 0.3850166621037557 |
| ] |
| }, |
| "escalation_oracle_error": { |
| "mean": 0.026657759662772857, |
| "sample_std": 0.004778017775719879, |
| "values": [ |
| 0.03202482287120588, |
| 0.022867237429594827, |
| 0.02508121868751787 |
| ] |
| }, |
| "department_brier": { |
| "mean": 0.005600015207012279, |
| "sample_std": 0.008240570162464452, |
| "values": [ |
| 0.0016986161765314043, |
| 3.464455186303505e-05, |
| 0.015066784892642397 |
| ] |
| }, |
| "department_accuracy": { |
| "mean": 0.9966666666666667, |
| "sample_std": 0.005773502691896262, |
| "values": [ |
| 1.0, |
| 1.0, |
| 0.99 |
| ] |
| } |
| }, |
| "global": { |
| "overall_brier": { |
| "mean": 0.09838776782360553, |
| "sample_std": 0.0009979802814574863, |
| "values": [ |
| 0.09768211345491773, |
| 0.09795160604976191, |
| 0.09952958396613695 |
| ] |
| }, |
| "overall_accuracy": { |
| "mean": 0.93375, |
| "sample_std": 0.0033071891388307306, |
| "values": [ |
| 0.9375, |
| 0.9325, |
| 0.93125 |
| ] |
| }, |
| "overall_nll": { |
| "mean": 0.14778804897965733, |
| "sample_std": 0.002297029210531731, |
| "values": [ |
| 0.14700351153971458, |
| 0.14598607091587076, |
| 0.15037456448338665 |
| ] |
| }, |
| "escalation_brier": { |
| "mean": 0.38652361728237167, |
| "sample_std": 0.003760700045117546, |
| "values": [ |
| 0.38865211513513864, |
| 0.3887373231295356, |
| 0.38218141358244084 |
| ] |
| }, |
| "escalation_oracle_error": { |
| "mean": 0.026428142371138913, |
| "sample_std": 0.0034560301384322215, |
| "values": [ |
| 0.028425395755713588, |
| 0.022437463121474682, |
| 0.02842156823622846 |
| ] |
| }, |
| "department_brier": { |
| "mean": 0.005771410536625953, |
| "sample_std": 0.008391051292519793, |
| "values": [ |
| 0.0018638307613938177, |
| 4.679844083130086e-05, |
| 0.01540360240765274 |
| ] |
| }, |
| "department_accuracy": { |
| "mean": 0.9966666666666667, |
| "sample_std": 0.005773502691896262, |
| "values": [ |
| 1.0, |
| 1.0, |
| 0.99 |
| ] |
| } |
| }, |
| "global_temperature": { |
| "mean": 1.132515534721732, |
| "sample_std": 0.07874133694372291, |
| "values": [ |
| 1.1481536214968828, |
| 1.0471285480508996, |
| 1.2022644346174132 |
| ] |
| } |
| }, |
| "supervised_control": { |
| "uncalibrated": { |
| "overall_brier": { |
| "mean": 0.09906243162091372, |
| "sample_std": 0.0009587909170585515, |
| "values": [ |
| 0.09887157277906948, |
| 0.0982134248610902, |
| 0.10010229722258146 |
| ] |
| }, |
| "overall_accuracy": { |
| "mean": 0.9316666666666666, |
| "sample_std": 0.0007216878364870169, |
| "values": [ |
| 0.93125, |
| 0.9325, |
| 0.93125 |
| ] |
| }, |
| "overall_nll": { |
| "mean": 0.14832740675427636, |
| "sample_std": 0.0025624895786719843, |
| "values": [ |
| 0.147861237853593, |
| 0.14603000355243925, |
| 0.15109097885679681 |
| ] |
| }, |
| "escalation_brier": { |
| "mean": 0.38980161907528416, |
| "sample_std": 0.004194049950854619, |
| "values": [ |
| 0.39154741751981503, |
| 0.3928407776022817, |
| 0.3850166621037557 |
| ] |
| }, |
| "escalation_oracle_error": { |
| "mean": 0.025821039346402664, |
| "sample_std": 0.0030412659264515717, |
| "values": [ |
| 0.02916396119279396, |
| 0.023217938158896164, |
| 0.02508121868751787 |
| ] |
| }, |
| "department_brier": { |
| "mean": 0.006305106290018521, |
| "sample_std": 0.007826417322748991, |
| "values": [ |
| 0.0038419606968017176, |
| 6.573280611448498e-06, |
| 0.015066784892642397 |
| ] |
| }, |
| "department_accuracy": { |
| "mean": 0.9950000000000001, |
| "sample_std": 0.0050000000000000044, |
| "values": [ |
| 0.995, |
| 1.0, |
| 0.99 |
| ] |
| } |
| }, |
| "global": { |
| "overall_brier": { |
| "mean": 0.09820905228437515, |
| "sample_std": 0.001176288146718124, |
| "values": [ |
| 0.09782410600728701, |
| 0.09727346687970145, |
| 0.09952958396613695 |
| ] |
| }, |
| "overall_accuracy": { |
| "mean": 0.9316666666666666, |
| "sample_std": 0.0007216878364870169, |
| "values": [ |
| 0.93125, |
| 0.9325, |
| 0.93125 |
| ] |
| }, |
| "overall_nll": { |
| "mean": 0.14726771800770008, |
| "sample_std": 0.0029072606023501507, |
| "values": [ |
| 0.1468155635573755, |
| 0.14461302598233813, |
| 0.15037456448338665 |
| ] |
| }, |
| "escalation_brier": { |
| "mean": 0.3861754575936172, |
| "sample_std": 0.0035728343752651046, |
| "values": [ |
| 0.387277574687684, |
| 0.389067384510727, |
| 0.38218141358244084 |
| ] |
| }, |
| "escalation_oracle_error": { |
| "mean": 0.027057448557831303, |
| "sample_std": 0.0035306814758794586, |
| "values": [ |
| 0.029702563918614765, |
| 0.023048213518650672, |
| 0.02842156823622846 |
| ] |
| }, |
| "department_brier": { |
| "mean": 0.006415403048229359, |
| "sample_std": 0.008014234084786671, |
| "values": [ |
| 0.003828435291642624, |
| 1.4171445392714965e-05, |
| 0.01540360240765274 |
| ] |
| }, |
| "department_accuracy": { |
| "mean": 0.9950000000000001, |
| "sample_std": 0.0050000000000000044, |
| "values": [ |
| 0.995, |
| 1.0, |
| 0.99 |
| ] |
| } |
| }, |
| "global_temperature": { |
| "mean": 1.1489654174191604, |
| "sample_std": 0.05289779127804991, |
| "values": [ |
| 1.1481536214968828, |
| 1.096478196143185, |
| 1.2022644346174132 |
| ] |
| } |
| } |
| }, |
| "selection": { |
| "17": { |
| "reinforce": { |
| "selected_epoch": 10, |
| "initial_weights_retained": false, |
| "updates_at_selected_checkpoint": 440, |
| "sigma_during_selected_epoch": 0.07599555414764668, |
| "checkpoint_sha256": "cd163f278f4b01dbf61e6dd73e070c7deab77eb359facaaaa70b85e916c3a258", |
| "training_seconds": 10.379978023999968 |
| }, |
| "pathwise": { |
| "selected_epoch": 2, |
| "initial_weights_retained": false, |
| "updates_at_selected_checkpoint": 88, |
| "sigma_during_selected_epoch": 0.40556541539484353, |
| "checkpoint_sha256": "c223ea54022e445f2fba036a4c06ffff311ea0533bc86aeba1a69df9c0a20112", |
| "training_seconds": 7.387697599999967 |
| }, |
| "supervised_control": { |
| "selected_epoch": 2, |
| "initial_weights_retained": false, |
| "updates_at_selected_checkpoint": 88, |
| "sigma_during_selected_epoch": null, |
| "checkpoint_sha256": "9053e81868e8679466b99b47e3bcbc2ce078dde1abd07add80db7d3b3c1fd85f", |
| "training_seconds": 7.166220393999993 |
| } |
| }, |
| "42": { |
| "reinforce": { |
| "selected_epoch": 9, |
| "initial_weights_retained": false, |
| "updates_at_selected_checkpoint": 396, |
| "sigma_during_selected_epoch": 0.0936908711430192, |
| "checkpoint_sha256": "3e06a89e16f13751f383c01e7ed473cb899fbfe9d1e3d2d7e9c7f2d5a4330088", |
| "training_seconds": 8.161688647999995 |
| }, |
| "pathwise": { |
| "selected_epoch": 6, |
| "initial_weights_retained": false, |
| "updates_at_selected_checkpoint": 264, |
| "sigma_during_selected_epoch": 0.17555958671075658, |
| "checkpoint_sha256": "b954683dcd070829ed54cad334ab0ee58e0b817d0cadc79256a7b900f32b3e24", |
| "training_seconds": 7.350335009000048 |
| }, |
| "supervised_control": { |
| "selected_epoch": 2, |
| "initial_weights_retained": false, |
| "updates_at_selected_checkpoint": 88, |
| "sigma_during_selected_epoch": null, |
| "checkpoint_sha256": "61b7a77120a14479c1a8c3e9d13289f224e1a843a34960a4b65cd932273a910d", |
| "training_seconds": 6.933952556000008 |
| } |
| }, |
| "83": { |
| "reinforce": { |
| "selected_epoch": 0, |
| "initial_weights_retained": true, |
| "updates_at_selected_checkpoint": 0, |
| "sigma_during_selected_epoch": null, |
| "checkpoint_sha256": "6e6c5eb650aa90fb660ab2255542bfca748b703d8e3ff39391c028aeca69f841", |
| "training_seconds": 7.734535555999969 |
| }, |
| "pathwise": { |
| "selected_epoch": 0, |
| "initial_weights_retained": true, |
| "updates_at_selected_checkpoint": 0, |
| "sigma_during_selected_epoch": null, |
| "checkpoint_sha256": "29e5e56b247bcda8629b64106d64386df926c8c151a49903ff7515ff09f6c02d", |
| "training_seconds": 7.240214808000019 |
| }, |
| "supervised_control": { |
| "selected_epoch": 0, |
| "initial_weights_retained": true, |
| "updates_at_selected_checkpoint": 0, |
| "sigma_during_selected_epoch": null, |
| "checkpoint_sha256": "3c5b5f5811cb899ed457d3eba2034e1f675ff2c1dd1537c1af8322df66e58d87", |
| "training_seconds": 6.97374545699995 |
| } |
| } |
| }, |
| "paired_by_seed": { |
| "17": { |
| "reinforce_minus_pathwise": { |
| "difference": "left minus right", |
| "cluster": "scenario_id", |
| "scenarios": 200, |
| "questions": 800, |
| "replicates": 2000, |
| "metrics": { |
| "accuracy": { |
| "estimate": -0.00625, |
| "ci95": [ |
| -0.0125, |
| -0.00125 |
| ] |
| }, |
| "brier": { |
| "estimate": 0.003913844936925385, |
| "ci95": [ |
| -0.0019083239478268218, |
| 0.010393815188188589 |
| ] |
| }, |
| "nll": { |
| "estimate": 0.006092518422554138, |
| "ci95": [ |
| -0.00299014435267016, |
| 0.016585367802551453 |
| ] |
| } |
| } |
| }, |
| "pathwise_minus_supervised_control": { |
| "difference": "left minus right", |
| "cluster": "scenario_id", |
| "scenarios": 200, |
| "questions": 800, |
| "replicates": 2000, |
| "metrics": { |
| "accuracy": { |
| "estimate": 0.00625, |
| "ci95": [ |
| 0.00125, |
| 0.0125 |
| ] |
| }, |
| "brier": { |
| "estimate": 0.000752540972895408, |
| "ci95": [ |
| -0.0019841229637799704, |
| 0.003491558453127027 |
| ] |
| }, |
| "nll": { |
| "estimate": 0.0023335834920879483, |
| "ci95": [ |
| -0.001855154584590223, |
| 0.006712404882488526 |
| ] |
| } |
| } |
| }, |
| "reinforce_minus_supervised_control": { |
| "difference": "left minus right", |
| "cluster": "scenario_id", |
| "scenarios": 200, |
| "questions": 800, |
| "replicates": 2000, |
| "metrics": { |
| "accuracy": { |
| "estimate": 0.0, |
| "ci95": [ |
| -0.00625, |
| 0.00625 |
| ] |
| }, |
| "brier": { |
| "estimate": 0.004666385909820787, |
| "ci95": [ |
| 0.0007259763022988612, |
| 0.009331068910668377 |
| ] |
| }, |
| "nll": { |
| "estimate": 0.008426101914642088, |
| "ci95": [ |
| 0.002169096358564351, |
| 0.016453444968806522 |
| ] |
| } |
| } |
| } |
| }, |
| "42": { |
| "reinforce_minus_pathwise": { |
| "difference": "left minus right", |
| "cluster": "scenario_id", |
| "scenarios": 200, |
| "questions": 800, |
| "replicates": 2000, |
| "metrics": { |
| "accuracy": { |
| "estimate": -0.0075, |
| "ci95": [ |
| -0.015, |
| -0.00125 |
| ] |
| }, |
| "brier": { |
| "estimate": 0.006709475175339073, |
| "ci95": [ |
| 0.0027681322291477563, |
| 0.010767941333516804 |
| ] |
| }, |
| "nll": { |
| "estimate": 0.011255448688946115, |
| "ci95": [ |
| 0.005433987932983549, |
| 0.017364808904972877 |
| ] |
| } |
| } |
| }, |
| "pathwise_minus_supervised_control": { |
| "difference": "left minus right", |
| "cluster": "scenario_id", |
| "scenarios": 200, |
| "questions": 800, |
| "replicates": 2000, |
| "metrics": { |
| "accuracy": { |
| "estimate": 0.0, |
| "ci95": [ |
| -0.005, |
| 0.005 |
| ] |
| }, |
| "brier": { |
| "estimate": 0.00027551657685014204, |
| "ci95": [ |
| -0.0014971832841162264, |
| 0.0024908339241938014 |
| ] |
| }, |
| "nll": { |
| "estimate": 0.000773115770650783, |
| "ci95": [ |
| -0.0017141946025765605, |
| 0.003720178160473578 |
| ] |
| } |
| } |
| }, |
| "reinforce_minus_supervised_control": { |
| "difference": "left minus right", |
| "cluster": "scenario_id", |
| "scenarios": 200, |
| "questions": 800, |
| "replicates": 2000, |
| "metrics": { |
| "accuracy": { |
| "estimate": -0.0075, |
| "ci95": [ |
| -0.01375, |
| -0.0025 |
| ] |
| }, |
| "brier": { |
| "estimate": 0.006984991752189208, |
| "ci95": [ |
| 0.003686490278951977, |
| 0.010600821256042716 |
| ] |
| }, |
| "nll": { |
| "estimate": 0.012028564459596882, |
| "ci95": [ |
| 0.006630782959270349, |
| 0.017939820278900245 |
| ] |
| } |
| } |
| } |
| }, |
| "83": { |
| "reinforce_minus_pathwise": { |
| "difference": "left minus right", |
| "cluster": "scenario_id", |
| "scenarios": 200, |
| "questions": 800, |
| "replicates": 2000, |
| "metrics": { |
| "accuracy": { |
| "estimate": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "brier": { |
| "estimate": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "nll": { |
| "estimate": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| } |
| } |
| }, |
| "pathwise_minus_supervised_control": { |
| "difference": "left minus right", |
| "cluster": "scenario_id", |
| "scenarios": 200, |
| "questions": 800, |
| "replicates": 2000, |
| "metrics": { |
| "accuracy": { |
| "estimate": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "brier": { |
| "estimate": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "nll": { |
| "estimate": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| } |
| } |
| }, |
| "reinforce_minus_supervised_control": { |
| "difference": "left minus right", |
| "cluster": "scenario_id", |
| "scenarios": 200, |
| "questions": 800, |
| "replicates": 2000, |
| "metrics": { |
| "accuracy": { |
| "estimate": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "brier": { |
| "estimate": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "nll": { |
| "estimate": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| } |
| } |
| } |
| } |
| }, |
| "validation_curves": { |
| "17": { |
| "reinforce": [ |
| 0.10525166365214286, |
| 0.12117375417225826, |
| 0.10056672435834922, |
| 0.12820932515977224, |
| 0.11795555333136519, |
| 0.13667396504524945, |
| 0.10247942364228006, |
| 0.1155518692646555, |
| 0.10779189981201505, |
| 0.10184562992595193, |
| 0.09559045068776588, |
| 0.10363244837789011, |
| 0.11131992410993118 |
| ], |
| "pathwise": [ |
| 0.10525166365214286, |
| 0.11065971989234902, |
| 0.09824757497096048, |
| 0.10657535849685988, |
| 0.10125011199781372, |
| 0.11510530208956354, |
| 0.1088730146333369, |
| 0.10679709966254954, |
| 0.1091377949351295, |
| 0.10441830290000402, |
| 0.10627546500851459, |
| 0.104999833073343, |
| 0.10231358827323535 |
| ], |
| "supervised_control": [ |
| 0.10525166365214286, |
| 0.1170103724525552, |
| 0.09544566093039751, |
| 0.10298943942544174, |
| 0.11113089717509701, |
| 0.10582123057009545, |
| 0.10669869198125166, |
| 0.11565393606109643, |
| 0.10731978267603598, |
| 0.10381348879637402, |
| 0.1051709352911545, |
| 0.10399307047526707, |
| 0.10163697996214222 |
| ] |
| }, |
| "42": { |
| "reinforce": [ |
| 0.1071913138430586, |
| 0.10377996122888916, |
| 0.10310212435470797, |
| 0.1049493730019816, |
| 0.1003548910545588, |
| 0.10705100129739843, |
| 0.10501917955379572, |
| 0.10237785883997365, |
| 0.10095500779253257, |
| 0.0975886054619946, |
| 0.09947045925175825, |
| 0.10244753233010398, |
| 0.12085995974123508 |
| ], |
| "pathwise": [ |
| 0.1071913138430586, |
| 0.10240869754659812, |
| 0.10712373726064847, |
| 0.11106727240314371, |
| 0.10569180604129055, |
| 0.11352077008710619, |
| 0.10221139976235721, |
| 0.10494969294374147, |
| 0.10567522695429225, |
| 0.10426949485443568, |
| 0.10381806706524084, |
| 0.1088538830265569, |
| 0.12025554920093047 |
| ], |
| "supervised_control": [ |
| 0.1071913138430586, |
| 0.10093467402595857, |
| 0.09649431923038009, |
| 0.0997434978569271, |
| 0.09846773433976905, |
| 0.1055279856369447, |
| 0.09749732962826968, |
| 0.09925536711833527, |
| 0.10009254807590924, |
| 0.0975074993691917, |
| 0.09881885361931592, |
| 0.10095188438534601, |
| 0.10791935218434073 |
| ] |
| }, |
| "83": { |
| "reinforce": [ |
| 0.09896155878389151, |
| 0.11207710387205463, |
| 0.10643482040185522, |
| 0.11008471482831858, |
| 0.15978935213367723, |
| 0.11076701087399243, |
| 0.11080816360842878, |
| 0.10352241231692563, |
| 0.10690213270748655, |
| 0.10033565235208199, |
| 0.1121528359071667, |
| 0.09959281776578198, |
| 0.1002893951963929 |
| ], |
| "pathwise": [ |
| 0.09896155878389151, |
| 0.10576785811327319, |
| 0.1119189157873618, |
| 0.10377035712901829, |
| 0.101134715685967, |
| 0.10383180125492152, |
| 0.10782662054546012, |
| 0.10116922768142117, |
| 0.10463641467225038, |
| 0.10265626522409704, |
| 0.10178678888427335, |
| 0.1039241634484645, |
| 0.10219550067863968 |
| ], |
| "supervised_control": [ |
| 0.09896155878389151, |
| 0.10096507843476553, |
| 0.10135653519125114, |
| 0.10307073141499946, |
| 0.10203415531357753, |
| 0.10494437655206518, |
| 0.10969654279162722, |
| 0.10307231324468878, |
| 0.1058754957502558, |
| 0.10402873280645988, |
| 0.10363822283071739, |
| 0.10643811765449535, |
| 0.10429820518581277 |
| ] |
| } |
| }, |
| "rerun_check": { |
| "17": { |
| "reinforce": { |
| "published_epoch": 10, |
| "rerun_epoch": 10, |
| "published_test_brier": 0.10353795868889026, |
| "rerun_test_brier": 0.10353795868889026, |
| "absolute_difference": 0.0, |
| "status": "identical_checkpoint" |
| }, |
| "supervised_control": { |
| "published_epoch": 2, |
| "rerun_epoch": 2, |
| "published_test_brier": 0.09887157277906948, |
| "rerun_test_brier": 0.09887157277906948, |
| "absolute_difference": 0.0, |
| "status": "identical_checkpoint" |
| } |
| }, |
| "42": { |
| "reinforce": { |
| "published_epoch": 9, |
| "rerun_epoch": 9, |
| "published_test_brier": 0.10519841661327942, |
| "rerun_test_brier": 0.10519841661327942, |
| "absolute_difference": 0.0, |
| "status": "identical_checkpoint" |
| }, |
| "supervised_control": { |
| "published_epoch": 2, |
| "rerun_epoch": 2, |
| "published_test_brier": 0.0982134248610902, |
| "rerun_test_brier": 0.0982134248610902, |
| "absolute_difference": 0.0, |
| "status": "identical_checkpoint" |
| } |
| }, |
| "83": { |
| "reinforce": { |
| "published_epoch": 0, |
| "rerun_epoch": 0, |
| "published_test_brier": 0.10010229722258146, |
| "rerun_test_brier": 0.10010229722258146, |
| "absolute_difference": 0.0, |
| "status": "identical_checkpoint" |
| }, |
| "supervised_control": { |
| "published_epoch": 0, |
| "rerun_epoch": 0, |
| "published_test_brier": 0.10010229722258146, |
| "rerun_test_brier": 0.10010229722258146, |
| "absolute_difference": 0.0, |
| "status": "identical_checkpoint" |
| } |
| } |
| }, |
| "gradient_diagnostics": { |
| "17": { |
| "0.5": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.46284149843625383, |
| "mean_norm": 0.6598389959377321, |
| "signal_to_noise": 1.0333015385554893, |
| "cosine_with_direct_brier": 0.9842502317484433, |
| "share_clipped_at_1": 0.2734375 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 0.00845221541550655, |
| "mean_norm": 0.6454276063316989, |
| "signal_to_noise": 85.8120535702609, |
| "cosine_with_direct_brier": 0.9905063608346889, |
| "share_clipped_at_1": 0.25 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 54.75978494195954, |
| "agreement_ratio_mean": 0.6462404959492155, |
| "agreement_ratio_max": 2.1452463817447716 |
| }, |
| "0.2": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.4869736717546126, |
| "mean_norm": 0.7193704744927283, |
| "signal_to_noise": 1.187323023967159, |
| "cosine_with_direct_brier": 0.9947243708989945, |
| "share_clipped_at_1": 0.26171875 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 0.0018503159757085983, |
| "mean_norm": 0.6971751033506695, |
| "signal_to_noise": 476.6724544984792, |
| "cosine_with_direct_brier": 0.9995299732998293, |
| "share_clipped_at_1": 0.25 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 263.18406053221304, |
| "agreement_ratio_mean": 0.3919386741168277, |
| "agreement_ratio_max": 0.58373299619882 |
| }, |
| "0.094": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.4063009218850315, |
| "mean_norm": 0.6971293234216475, |
| "signal_to_noise": 1.2856995562457865, |
| "cosine_with_direct_brier": 0.9912163844156276, |
| "share_clipped_at_1": 0.24609375 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 0.0005232238525682875, |
| "mean_norm": 0.7062457392591972, |
| "signal_to_noise": 1939.8987845674772, |
| "cosine_with_direct_brier": 0.9999787942523685, |
| "share_clipped_at_1": 0.25 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 776.5336383092434, |
| "agreement_ratio_mean": 0.6574977694156234, |
| "agreement_ratio_max": 0.8380482957796178 |
| }, |
| "0.076": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.36551461172232724, |
| "mean_norm": 0.6674477134250594, |
| "signal_to_noise": 1.283015108650251, |
| "cosine_with_direct_brier": 0.9870454019371077, |
| "share_clipped_at_1": 0.2421875 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 0.00028707658877636435, |
| "mean_norm": 0.7080219181693501, |
| "signal_to_noise": 3451.956578238144, |
| "cosine_with_direct_brier": 0.9999929331939208, |
| "share_clipped_at_1": 0.25 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 1273.2303016428377, |
| "agreement_ratio_mean": 1.5013428312808506, |
| "agreement_ratio_max": 2.603263457853887 |
| }, |
| "0.05": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.4782464320196254, |
| "mean_norm": 0.7189984610166613, |
| "signal_to_noise": 1.1929225607573308, |
| "cosine_with_direct_brier": 0.996488594859608, |
| "share_clipped_at_1": 0.2421875 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 0.00010644368043091452, |
| "mean_norm": 0.7091560970193896, |
| "signal_to_noise": 8910.709102980092, |
| "cosine_with_direct_brier": 0.9999980663323704, |
| "share_clipped_at_1": 0.25 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 4492.952799861361, |
| "agreement_ratio_mean": 0.29347333854653, |
| "agreement_ratio_max": 0.45492176428585107 |
| } |
| }, |
| "42": { |
| "0.5": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.2771912973195207, |
| "mean_norm": 0.592492117214848, |
| "signal_to_noise": 1.9141604026010564, |
| "cosine_with_direct_brier": 0.9747485685830328, |
| "share_clipped_at_1": 0.19921875 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 0.0033385613311176666, |
| "mean_norm": 0.5834500297148322, |
| "signal_to_noise": 142.04900848674697, |
| "cosine_with_direct_brier": 0.9892566932956796, |
| "share_clipped_at_1": 0.0 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 83.02716943849703, |
| "agreement_ratio_mean": 0.6783879745896075, |
| "agreement_ratio_max": 1.5186668911979924 |
| }, |
| "0.2": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.2884170785880069, |
| "mean_norm": 0.6226456908829614, |
| "signal_to_noise": 1.9367242726579605, |
| "cosine_with_direct_brier": 0.9940841985439475, |
| "share_clipped_at_1": 0.1875 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 0.0007028900444951943, |
| "mean_norm": 0.6222810613493973, |
| "signal_to_noise": 687.1329998114431, |
| "cosine_with_direct_brier": 0.9995022068419931, |
| "share_clipped_at_1": 0.0 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 410.3302939724292, |
| "agreement_ratio_mean": 0.26361974270296656, |
| "agreement_ratio_max": 0.5928757461945089 |
| }, |
| "0.094": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.25145807594360137, |
| "mean_norm": 0.6630858096857781, |
| "signal_to_noise": 2.374004057848581, |
| "cosine_with_direct_brier": 0.9927662324227045, |
| "share_clipped_at_1": 0.234375 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 0.00016147539780585437, |
| "mean_norm": 0.6251221497057325, |
| "signal_to_noise": 3278.724070568899, |
| "cosine_with_direct_brier": 0.9999844544771266, |
| "share_clipped_at_1": 0.0 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 1557.253175161304, |
| "agreement_ratio_mean": 1.1869134887072454, |
| "agreement_ratio_max": 2.6694703713187025 |
| }, |
| "0.076": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.2478367855231179, |
| "mean_norm": 0.6313738281380668, |
| "signal_to_noise": 2.635549391461546, |
| "cosine_with_direct_brier": 0.9925094055317731, |
| "share_clipped_at_1": 0.22265625 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 0.00010725349941810217, |
| "mean_norm": 0.6267993771556946, |
| "signal_to_noise": 5298.541434061852, |
| "cosine_with_direct_brier": 0.9999383321706993, |
| "share_clipped_at_1": 0.0 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 2310.7571022646575, |
| "agreement_ratio_mean": 0.7507869621645225, |
| "agreement_ratio_max": 1.6653046768007829 |
| }, |
| "0.05": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.2580779727844232, |
| "mean_norm": 0.618081486326139, |
| "signal_to_noise": 2.135000961425198, |
| "cosine_with_direct_brier": 0.9920991519826978, |
| "share_clipped_at_1": 0.203125 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 3.900023669546475e-05, |
| "mean_norm": 0.6270679357144233, |
| "signal_to_noise": 13268.571913919826, |
| "cosine_with_direct_brier": 0.9999984252978076, |
| "share_clipped_at_1": 0.0 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 6617.343756132498, |
| "agreement_ratio_mean": 0.4291527642400812, |
| "agreement_ratio_max": 0.47181178504984306 |
| } |
| }, |
| "83": { |
| "0.5": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.9379789491702113, |
| "mean_norm": 1.0131259212989498, |
| "signal_to_noise": 1.2294118280724042, |
| "cosine_with_direct_brier": 0.9716701540952857, |
| "share_clipped_at_1": 0.46484375 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 0.021000866994369864, |
| "mean_norm": 0.9711555070997098, |
| "signal_to_noise": 62.50398083635155, |
| "cosine_with_direct_brier": 0.9916613115070596, |
| "share_clipped_at_1": 0.30859375 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 44.66382028045198, |
| "agreement_ratio_mean": 1.0390317528476336, |
| "agreement_ratio_max": 3.1108105745265986 |
| }, |
| "0.2": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.9502030822499791, |
| "mean_norm": 1.0477055256799697, |
| "signal_to_noise": 1.2896964811368419, |
| "cosine_with_direct_brier": 0.989264239600379, |
| "share_clipped_at_1": 0.484375 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 0.0047728485235824375, |
| "mean_norm": 1.0074089560205581, |
| "signal_to_noise": 305.50640483071186, |
| "cosine_with_direct_brier": 0.9996899365099464, |
| "share_clipped_at_1": 0.28125 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 199.08511186874398, |
| "agreement_ratio_mean": 0.5198028422450968, |
| "agreement_ratio_max": 1.0144624628629946 |
| }, |
| "0.094": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.8118193318948566, |
| "mean_norm": 1.0178130705439359, |
| "signal_to_noise": 1.376644686625821, |
| "cosine_with_direct_brier": 0.9859072883477893, |
| "share_clipped_at_1": 0.37890625 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 0.00138759177937646, |
| "mean_norm": 1.0092563878962901, |
| "signal_to_noise": 1222.740641309625, |
| "cosine_with_direct_brier": 0.9999867472502315, |
| "share_clipped_at_1": 0.25390625 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 585.0563140837161, |
| "agreement_ratio_mean": 0.6269024917446663, |
| "agreement_ratio_max": 0.831940831600837 |
| }, |
| "0.076": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.7258680385626702, |
| "mean_norm": 0.9482305821135979, |
| "signal_to_noise": 1.2840918728099542, |
| "cosine_with_direct_brier": 0.9833008012590074, |
| "share_clipped_at_1": 0.40625 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 0.0007954545657276127, |
| "mean_norm": 1.0049935327589117, |
| "signal_to_noise": 1947.9007735808582, |
| "cosine_with_direct_brier": 0.999988541600715, |
| "share_clipped_at_1": 0.25 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 912.519796650748, |
| "agreement_ratio_mean": 1.2886335785701934, |
| "agreement_ratio_max": 2.4596769321546135 |
| }, |
| "0.05": { |
| "reinforce": { |
| "trace_variance_per_estimate": 0.9184363455696682, |
| "mean_norm": 1.018317408106631, |
| "signal_to_noise": 1.2225270228346186, |
| "cosine_with_direct_brier": 0.9963402226443532, |
| "share_clipped_at_1": 0.3984375 |
| }, |
| "pathwise": { |
| "trace_variance_per_estimate": 0.0003136667891719711, |
| "mean_norm": 1.0067536202615717, |
| "signal_to_noise": 4997.685269854763, |
| "cosine_with_direct_brier": 0.999998956799107, |
| "share_clipped_at_1": 0.25 |
| }, |
| "variance_ratio_reinforce_over_pathwise": 2928.0637200839456, |
| "agreement_ratio_mean": 0.22823046877984618, |
| "agreement_ratio_max": 0.29483305814163796 |
| } |
| } |
| }, |
| "noise_replicates": { |
| "reinforce": { |
| "per_seed": { |
| "17": { |
| "draws": [ |
| { |
| "draw": "main", |
| "test_brier": 0.10353795868889026, |
| "selected_epoch": 10 |
| }, |
| { |
| "draw": "noise1001", |
| "test_brier": 0.09958477971636065, |
| "selected_epoch": 2 |
| }, |
| { |
| "draw": "noise1002", |
| "test_brier": 0.0943575117178091, |
| "selected_epoch": 9 |
| }, |
| { |
| "draw": "noise1003", |
| "test_brier": 0.10885462190777247, |
| "selected_epoch": 4 |
| } |
| ], |
| "test_brier": { |
| "mean": 0.10158371800770812, |
| "sample_std": 0.006134571753614905, |
| "values": [ |
| 0.10353795868889026, |
| 0.09958477971636065, |
| 0.0943575117178091, |
| 0.10885462190777247 |
| ] |
| } |
| }, |
| "42": { |
| "draws": [ |
| { |
| "draw": "main", |
| "test_brier": 0.10519841661327942, |
| "selected_epoch": 9 |
| }, |
| { |
| "draw": "noise1001", |
| "test_brier": 0.10397147871503436, |
| "selected_epoch": 3 |
| }, |
| { |
| "draw": "noise1002", |
| "test_brier": 0.09861655845798437, |
| "selected_epoch": 4 |
| }, |
| { |
| "draw": "noise1003", |
| "test_brier": 0.10681786461917256, |
| "selected_epoch": 3 |
| } |
| ], |
| "test_brier": { |
| "mean": 0.10365107960136766, |
| "sample_std": 0.003553019275331776, |
| "values": [ |
| 0.10519841661327942, |
| 0.10397147871503436, |
| 0.09861655845798437, |
| 0.10681786461917256 |
| ] |
| } |
| }, |
| "83": { |
| "draws": [ |
| { |
| "draw": "main", |
| "test_brier": 0.10010229722258146, |
| "selected_epoch": 0 |
| }, |
| { |
| "draw": "noise1001", |
| "test_brier": 0.10277142610033113, |
| "selected_epoch": 11 |
| }, |
| { |
| "draw": "noise1002", |
| "test_brier": 0.09628757504424384, |
| "selected_epoch": 10 |
| }, |
| { |
| "draw": "noise1003", |
| "test_brier": 0.09707360043260937, |
| "selected_epoch": 7 |
| } |
| ], |
| "test_brier": { |
| "mean": 0.09905872469994145, |
| "sample_std": 0.002971710173704046, |
| "values": [ |
| 0.10010229722258146, |
| 0.10277142610033113, |
| 0.09628757504424384, |
| 0.09707360043260937 |
| ] |
| } |
| } |
| }, |
| "pooled_test_brier": { |
| "mean": 0.10143117410300574, |
| "sample_std": 0.004467895566466421, |
| "values": [ |
| 0.10353795868889026, |
| 0.09958477971636065, |
| 0.0943575117178091, |
| 0.10885462190777247, |
| 0.10519841661327942, |
| 0.10397147871503436, |
| 0.09861655845798437, |
| 0.10681786461917256, |
| 0.10010229722258146, |
| 0.10277142610033113, |
| 0.09628757504424384, |
| 0.09707360043260937 |
| ] |
| } |
| }, |
| "pathwise": { |
| "per_seed": { |
| "17": { |
| "draws": [ |
| { |
| "draw": "main", |
| "test_brier": 0.09962411375196488, |
| "selected_epoch": 2 |
| }, |
| { |
| "draw": "noise1001", |
| "test_brier": 0.09961351868543376, |
| "selected_epoch": 12 |
| }, |
| { |
| "draw": "noise1002", |
| "test_brier": 0.10099567585955302, |
| "selected_epoch": 2 |
| }, |
| { |
| "draw": "noise1003", |
| "test_brier": 0.0962117533291581, |
| "selected_epoch": 3 |
| } |
| ], |
| "test_brier": { |
| "mean": 0.09911126540652744, |
| "sample_std": 0.0020390720784932872, |
| "values": [ |
| 0.09962411375196488, |
| 0.09961351868543376, |
| 0.10099567585955302, |
| 0.0962117533291581 |
| ] |
| } |
| }, |
| "42": { |
| "draws": [ |
| { |
| "draw": "main", |
| "test_brier": 0.09848894143794036, |
| "selected_epoch": 6 |
| }, |
| { |
| "draw": "noise1001", |
| "test_brier": 0.1000946447392786, |
| "selected_epoch": 9 |
| }, |
| { |
| "draw": "noise1002", |
| "test_brier": 0.10004436691942789, |
| "selected_epoch": 9 |
| }, |
| { |
| "draw": "noise1003", |
| "test_brier": 0.09858985556282747, |
| "selected_epoch": 4 |
| } |
| ], |
| "test_brier": { |
| "mean": 0.09930445216486856, |
| "sample_std": 0.0008846061643562034, |
| "values": [ |
| 0.09848894143794036, |
| 0.1000946447392786, |
| 0.10004436691942789, |
| 0.09858985556282747 |
| ] |
| } |
| }, |
| "83": { |
| "draws": [ |
| { |
| "draw": "main", |
| "test_brier": 0.10010229722258146, |
| "selected_epoch": 0 |
| }, |
| { |
| "draw": "noise1001", |
| "test_brier": 0.10010229722258146, |
| "selected_epoch": 0 |
| }, |
| { |
| "draw": "noise1002", |
| "test_brier": 0.10010229722258146, |
| "selected_epoch": 0 |
| }, |
| { |
| "draw": "noise1003", |
| "test_brier": 0.09929478719071466, |
| "selected_epoch": 4 |
| } |
| ], |
| "test_brier": { |
| "mean": 0.09990041971461476, |
| "sample_std": 0.00040375501593339913, |
| "values": [ |
| 0.10010229722258146, |
| 0.10010229722258146, |
| 0.10010229722258146, |
| 0.09929478719071466 |
| ] |
| } |
| } |
| }, |
| "pooled_test_brier": { |
| "mean": 0.09943871242867025, |
| "sample_std": 0.0012308078394866614, |
| "values": [ |
| 0.09962411375196488, |
| 0.09961351868543376, |
| 0.10099567585955302, |
| 0.0962117533291581, |
| 0.09848894143794036, |
| 0.1000946447392786, |
| 0.10004436691942789, |
| 0.09858985556282747, |
| 0.10010229722258146, |
| 0.10010229722258146, |
| 0.10010229722258146, |
| 0.09929478719071466 |
| ] |
| } |
| }, |
| "supervised_control_test_brier": { |
| "mean": 0.09906243162091372, |
| "sample_std": 0.0009587909170585515, |
| "values": [ |
| 0.09887157277906948, |
| 0.0982134248610902, |
| 0.10010229722258146 |
| ] |
| } |
| } |
| } |
