{
  "scope": "Exploratory: these test splits have previously informed development.",
  "seeds": [
    17,
    42,
    83
  ],
  "torch": "2.11.0+cu128",
  "device": "cuda:0",
  "arms": {
    "reinforce": {
      "uncalibrated": {
        "overall_brier": {
          "mean": 0.10294622417491706,
          "sample_std": 0.002599080705236424,
          "values": [
            0.10353795868889026,
            0.10519841661327942,
            0.10010229722258146
          ]
        },
        "overall_accuracy": {
          "mean": 0.9291666666666667,
          "sample_std": 0.0036084391824351483,
          "values": [
            0.93125,
            0.925,
            0.93125
          ]
        },
        "overall_nll": {
          "mean": 0.1551456288790227,
          "sample_std": 0.0036213881972196487,
          "values": [
            0.15628733976823508,
            0.15805856801203613,
            0.15109097885679681
          ]
        },
        "escalation_brier": {
          "mean": 0.39582496485276236,
          "sample_std": 0.015027460780644108,
          "values": [
            0.3894728404020668,
            0.4129853920524647,
            0.3850166621037557
          ]
        },
        "escalation_oracle_error": {
          "mean": 0.029907245868575776,
          "sample_std": 0.0054552554318909674,
          "values": [
            0.028814277704596627,
            0.03582624121361282,
            0.02508121868751787
          ]
        },
        "department_brier": {
          "mean": 0.015501622589877693,
          "sample_std": 0.008755121273280025,
          "values": [
            0.02446606011017072,
            0.006972022766819956,
            0.015066784892642397
          ]
        },
        "department_accuracy": {
          "mean": 0.9883333333333333,
          "sample_std": 0.00763762615825974,
          "values": [
            0.98,
            0.995,
            0.99
          ]
        }
      },
      "global": {
        "overall_brier": {
          "mean": 0.10175951933231486,
          "sample_std": 0.0019328278968226429,
          "values": [
            0.1029542671867347,
            0.10279470684407294,
            0.09952958396613695
          ]
        },
        "overall_accuracy": {
          "mean": 0.9291666666666667,
          "sample_std": 0.0036084391824351483,
          "values": [
            0.93125,
            0.925,
            0.93125
          ]
        },
        "overall_nll": {
          "mean": 0.15411299632607459,
          "sample_std": 0.003556021658733511,
          "values": [
            0.15745305743150984,
            0.15451136706332724,
            0.15037456448338665
          ]
        },
        "escalation_brier": {
          "mean": 0.3900226244591283,
          "sample_std": 0.010218525967785686,
          "values": [
            0.38630746982104214,
            0.4015789899739018,
            0.38218141358244084
          ]
        },
        "escalation_oracle_error": {
          "mean": 0.032096277186036824,
          "sample_std": 0.003198074435346083,
          "values": [
            0.03361730003959572,
            0.03424996328228628,
            0.02842156823622846
          ]
        },
        "department_brier": {
          "mean": 0.016163031713855146,
          "sample_std": 0.008499735845706667,
          "values": [
            0.0250169991241882,
            0.008068493609724506,
            0.01540360240765274
          ]
        },
        "department_accuracy": {
          "mean": 0.9883333333333333,
          "sample_std": 0.00763762615825974,
          "values": [
            0.98,
            0.995,
            0.99
          ]
        }
      },
      "global_temperature": {
        "mean": 1.240038419401916,
        "sample_std": 0.03271323042554636,
        "values": [
          1.2589254117941675,
          1.2589254117941675,
          1.2022644346174132
        ]
      }
    },
    "pathwise": {
      "uncalibrated": {
        "overall_brier": {
          "mean": 0.0994051174708289,
          "sample_std": 0.000828672884992785,
          "values": [
            0.09962411375196488,
            0.09848894143794036,
            0.10010229722258146
          ]
        },
        "overall_accuracy": {
          "mean": 0.93375,
          "sample_std": 0.0033071891388307306,
          "values": [
            0.9375,
            0.9325,
            0.93125
          ]
        },
        "overall_nll": {
          "mean": 0.1493629731751893,
          "sample_std": 0.002261727963142961,
          "values": [
            0.15019482134568096,
            0.14680311932309004,
            0.15109097885679681
          ]
        },
        "escalation_brier": {
          "mean": 0.39087688054770786,
          "sample_std": 0.005834194495826293,
          "values": [
            0.3966846981039322,
            0.39092928143543576,
            0.3850166621037557
          ]
        },
        "escalation_oracle_error": {
          "mean": 0.026657759662772857,
          "sample_std": 0.004778017775719879,
          "values": [
            0.03202482287120588,
            0.022867237429594827,
            0.02508121868751787
          ]
        },
        "department_brier": {
          "mean": 0.005600015207012279,
          "sample_std": 0.008240570162464452,
          "values": [
            0.0016986161765314043,
            3.464455186303505e-05,
            0.015066784892642397
          ]
        },
        "department_accuracy": {
          "mean": 0.9966666666666667,
          "sample_std": 0.005773502691896262,
          "values": [
            1.0,
            1.0,
            0.99
          ]
        }
      },
      "global": {
        "overall_brier": {
          "mean": 0.09838776782360553,
          "sample_std": 0.0009979802814574863,
          "values": [
            0.09768211345491773,
            0.09795160604976191,
            0.09952958396613695
          ]
        },
        "overall_accuracy": {
          "mean": 0.93375,
          "sample_std": 0.0033071891388307306,
          "values": [
            0.9375,
            0.9325,
            0.93125
          ]
        },
        "overall_nll": {
          "mean": 0.14778804897965733,
          "sample_std": 0.002297029210531731,
          "values": [
            0.14700351153971458,
            0.14598607091587076,
            0.15037456448338665
          ]
        },
        "escalation_brier": {
          "mean": 0.38652361728237167,
          "sample_std": 0.003760700045117546,
          "values": [
            0.38865211513513864,
            0.3887373231295356,
            0.38218141358244084
          ]
        },
        "escalation_oracle_error": {
          "mean": 0.026428142371138913,
          "sample_std": 0.0034560301384322215,
          "values": [
            0.028425395755713588,
            0.022437463121474682,
            0.02842156823622846
          ]
        },
        "department_brier": {
          "mean": 0.005771410536625953,
          "sample_std": 0.008391051292519793,
          "values": [
            0.0018638307613938177,
            4.679844083130086e-05,
            0.01540360240765274
          ]
        },
        "department_accuracy": {
          "mean": 0.9966666666666667,
          "sample_std": 0.005773502691896262,
          "values": [
            1.0,
            1.0,
            0.99
          ]
        }
      },
      "global_temperature": {
        "mean": 1.132515534721732,
        "sample_std": 0.07874133694372291,
        "values": [
          1.1481536214968828,
          1.0471285480508996,
          1.2022644346174132
        ]
      }
    },
    "supervised_control": {
      "uncalibrated": {
        "overall_brier": {
          "mean": 0.09906243162091372,
          "sample_std": 0.0009587909170585515,
          "values": [
            0.09887157277906948,
            0.0982134248610902,
            0.10010229722258146
          ]
        },
        "overall_accuracy": {
          "mean": 0.9316666666666666,
          "sample_std": 0.0007216878364870169,
          "values": [
            0.93125,
            0.9325,
            0.93125
          ]
        },
        "overall_nll": {
          "mean": 0.14832740675427636,
          "sample_std": 0.0025624895786719843,
          "values": [
            0.147861237853593,
            0.14603000355243925,
            0.15109097885679681
          ]
        },
        "escalation_brier": {
          "mean": 0.38980161907528416,
          "sample_std": 0.004194049950854619,
          "values": [
            0.39154741751981503,
            0.3928407776022817,
            0.3850166621037557
          ]
        },
        "escalation_oracle_error": {
          "mean": 0.025821039346402664,
          "sample_std": 0.0030412659264515717,
          "values": [
            0.02916396119279396,
            0.023217938158896164,
            0.02508121868751787
          ]
        },
        "department_brier": {
          "mean": 0.006305106290018521,
          "sample_std": 0.007826417322748991,
          "values": [
            0.0038419606968017176,
            6.573280611448498e-06,
            0.015066784892642397
          ]
        },
        "department_accuracy": {
          "mean": 0.9950000000000001,
          "sample_std": 0.0050000000000000044,
          "values": [
            0.995,
            1.0,
            0.99
          ]
        }
      },
      "global": {
        "overall_brier": {
          "mean": 0.09820905228437515,
          "sample_std": 0.001176288146718124,
          "values": [
            0.09782410600728701,
            0.09727346687970145,
            0.09952958396613695
          ]
        },
        "overall_accuracy": {
          "mean": 0.9316666666666666,
          "sample_std": 0.0007216878364870169,
          "values": [
            0.93125,
            0.9325,
            0.93125
          ]
        },
        "overall_nll": {
          "mean": 0.14726771800770008,
          "sample_std": 0.0029072606023501507,
          "values": [
            0.1468155635573755,
            0.14461302598233813,
            0.15037456448338665
          ]
        },
        "escalation_brier": {
          "mean": 0.3861754575936172,
          "sample_std": 0.0035728343752651046,
          "values": [
            0.387277574687684,
            0.389067384510727,
            0.38218141358244084
          ]
        },
        "escalation_oracle_error": {
          "mean": 0.027057448557831303,
          "sample_std": 0.0035306814758794586,
          "values": [
            0.029702563918614765,
            0.023048213518650672,
            0.02842156823622846
          ]
        },
        "department_brier": {
          "mean": 0.006415403048229359,
          "sample_std": 0.008014234084786671,
          "values": [
            0.003828435291642624,
            1.4171445392714965e-05,
            0.01540360240765274
          ]
        },
        "department_accuracy": {
          "mean": 0.9950000000000001,
          "sample_std": 0.0050000000000000044,
          "values": [
            0.995,
            1.0,
            0.99
          ]
        }
      },
      "global_temperature": {
        "mean": 1.1489654174191604,
        "sample_std": 0.05289779127804991,
        "values": [
          1.1481536214968828,
          1.096478196143185,
          1.2022644346174132
        ]
      }
    }
  },
  "selection": {
    "17": {
      "reinforce": {
        "selected_epoch": 10,
        "initial_weights_retained": false,
        "updates_at_selected_checkpoint": 440,
        "sigma_during_selected_epoch": 0.07599555414764668,
        "checkpoint_sha256": "cd163f278f4b01dbf61e6dd73e070c7deab77eb359facaaaa70b85e916c3a258",
        "training_seconds": 10.379978023999968
      },
      "pathwise": {
        "selected_epoch": 2,
        "initial_weights_retained": false,
        "updates_at_selected_checkpoint": 88,
        "sigma_during_selected_epoch": 0.40556541539484353,
        "checkpoint_sha256": "c223ea54022e445f2fba036a4c06ffff311ea0533bc86aeba1a69df9c0a20112",
        "training_seconds": 7.387697599999967
      },
      "supervised_control": {
        "selected_epoch": 2,
        "initial_weights_retained": false,
        "updates_at_selected_checkpoint": 88,
        "sigma_during_selected_epoch": null,
        "checkpoint_sha256": "9053e81868e8679466b99b47e3bcbc2ce078dde1abd07add80db7d3b3c1fd85f",
        "training_seconds": 7.166220393999993
      }
    },
    "42": {
      "reinforce": {
        "selected_epoch": 9,
        "initial_weights_retained": false,
        "updates_at_selected_checkpoint": 396,
        "sigma_during_selected_epoch": 0.0936908711430192,
        "checkpoint_sha256": "3e06a89e16f13751f383c01e7ed473cb899fbfe9d1e3d2d7e9c7f2d5a4330088",
        "training_seconds": 8.161688647999995
      },
      "pathwise": {
        "selected_epoch": 6,
        "initial_weights_retained": false,
        "updates_at_selected_checkpoint": 264,
        "sigma_during_selected_epoch": 0.17555958671075658,
        "checkpoint_sha256": "b954683dcd070829ed54cad334ab0ee58e0b817d0cadc79256a7b900f32b3e24",
        "training_seconds": 7.350335009000048
      },
      "supervised_control": {
        "selected_epoch": 2,
        "initial_weights_retained": false,
        "updates_at_selected_checkpoint": 88,
        "sigma_during_selected_epoch": null,
        "checkpoint_sha256": "61b7a77120a14479c1a8c3e9d13289f224e1a843a34960a4b65cd932273a910d",
        "training_seconds": 6.933952556000008
      }
    },
    "83": {
      "reinforce": {
        "selected_epoch": 0,
        "initial_weights_retained": true,
        "updates_at_selected_checkpoint": 0,
        "sigma_during_selected_epoch": null,
        "checkpoint_sha256": "6e6c5eb650aa90fb660ab2255542bfca748b703d8e3ff39391c028aeca69f841",
        "training_seconds": 7.734535555999969
      },
      "pathwise": {
        "selected_epoch": 0,
        "initial_weights_retained": true,
        "updates_at_selected_checkpoint": 0,
        "sigma_during_selected_epoch": null,
        "checkpoint_sha256": "29e5e56b247bcda8629b64106d64386df926c8c151a49903ff7515ff09f6c02d",
        "training_seconds": 7.240214808000019
      },
      "supervised_control": {
        "selected_epoch": 0,
        "initial_weights_retained": true,
        "updates_at_selected_checkpoint": 0,
        "sigma_during_selected_epoch": null,
        "checkpoint_sha256": "3c5b5f5811cb899ed457d3eba2034e1f675ff2c1dd1537c1af8322df66e58d87",
        "training_seconds": 6.97374545699995
      }
    }
  },
  "paired_by_seed": {
    "17": {
      "reinforce_minus_pathwise": {
        "difference": "left minus right",
        "cluster": "scenario_id",
        "scenarios": 200,
        "questions": 800,
        "replicates": 2000,
        "metrics": {
          "accuracy": {
            "estimate": -0.00625,
            "ci95": [
              -0.0125,
              -0.00125
            ]
          },
          "brier": {
            "estimate": 0.003913844936925385,
            "ci95": [
              -0.0019083239478268218,
              0.010393815188188589
            ]
          },
          "nll": {
            "estimate": 0.006092518422554138,
            "ci95": [
              -0.00299014435267016,
              0.016585367802551453
            ]
          }
        }
      },
      "pathwise_minus_supervised_control": {
        "difference": "left minus right",
        "cluster": "scenario_id",
        "scenarios": 200,
        "questions": 800,
        "replicates": 2000,
        "metrics": {
          "accuracy": {
            "estimate": 0.00625,
            "ci95": [
              0.00125,
              0.0125
            ]
          },
          "brier": {
            "estimate": 0.000752540972895408,
            "ci95": [
              -0.0019841229637799704,
              0.003491558453127027
            ]
          },
          "nll": {
            "estimate": 0.0023335834920879483,
            "ci95": [
              -0.001855154584590223,
              0.006712404882488526
            ]
          }
        }
      },
      "reinforce_minus_supervised_control": {
        "difference": "left minus right",
        "cluster": "scenario_id",
        "scenarios": 200,
        "questions": 800,
        "replicates": 2000,
        "metrics": {
          "accuracy": {
            "estimate": 0.0,
            "ci95": [
              -0.00625,
              0.00625
            ]
          },
          "brier": {
            "estimate": 0.004666385909820787,
            "ci95": [
              0.0007259763022988612,
              0.009331068910668377
            ]
          },
          "nll": {
            "estimate": 0.008426101914642088,
            "ci95": [
              0.002169096358564351,
              0.016453444968806522
            ]
          }
        }
      }
    },
    "42": {
      "reinforce_minus_pathwise": {
        "difference": "left minus right",
        "cluster": "scenario_id",
        "scenarios": 200,
        "questions": 800,
        "replicates": 2000,
        "metrics": {
          "accuracy": {
            "estimate": -0.0075,
            "ci95": [
              -0.015,
              -0.00125
            ]
          },
          "brier": {
            "estimate": 0.006709475175339073,
            "ci95": [
              0.0027681322291477563,
              0.010767941333516804
            ]
          },
          "nll": {
            "estimate": 0.011255448688946115,
            "ci95": [
              0.005433987932983549,
              0.017364808904972877
            ]
          }
        }
      },
      "pathwise_minus_supervised_control": {
        "difference": "left minus right",
        "cluster": "scenario_id",
        "scenarios": 200,
        "questions": 800,
        "replicates": 2000,
        "metrics": {
          "accuracy": {
            "estimate": 0.0,
            "ci95": [
              -0.005,
              0.005
            ]
          },
          "brier": {
            "estimate": 0.00027551657685014204,
            "ci95": [
              -0.0014971832841162264,
              0.0024908339241938014
            ]
          },
          "nll": {
            "estimate": 0.000773115770650783,
            "ci95": [
              -0.0017141946025765605,
              0.003720178160473578
            ]
          }
        }
      },
      "reinforce_minus_supervised_control": {
        "difference": "left minus right",
        "cluster": "scenario_id",
        "scenarios": 200,
        "questions": 800,
        "replicates": 2000,
        "metrics": {
          "accuracy": {
            "estimate": -0.0075,
            "ci95": [
              -0.01375,
              -0.0025
            ]
          },
          "brier": {
            "estimate": 0.006984991752189208,
            "ci95": [
              0.003686490278951977,
              0.010600821256042716
            ]
          },
          "nll": {
            "estimate": 0.012028564459596882,
            "ci95": [
              0.006630782959270349,
              0.017939820278900245
            ]
          }
        }
      }
    },
    "83": {
      "reinforce_minus_pathwise": {
        "difference": "left minus right",
        "cluster": "scenario_id",
        "scenarios": 200,
        "questions": 800,
        "replicates": 2000,
        "metrics": {
          "accuracy": {
            "estimate": 0.0,
            "ci95": [
              0.0,
              0.0
            ]
          },
          "brier": {
            "estimate": 0.0,
            "ci95": [
              0.0,
              0.0
            ]
          },
          "nll": {
            "estimate": 0.0,
            "ci95": [
              0.0,
              0.0
            ]
          }
        }
      },
      "pathwise_minus_supervised_control": {
        "difference": "left minus right",
        "cluster": "scenario_id",
        "scenarios": 200,
        "questions": 800,
        "replicates": 2000,
        "metrics": {
          "accuracy": {
            "estimate": 0.0,
            "ci95": [
              0.0,
              0.0
            ]
          },
          "brier": {
            "estimate": 0.0,
            "ci95": [
              0.0,
              0.0
            ]
          },
          "nll": {
            "estimate": 0.0,
            "ci95": [
              0.0,
              0.0
            ]
          }
        }
      },
      "reinforce_minus_supervised_control": {
        "difference": "left minus right",
        "cluster": "scenario_id",
        "scenarios": 200,
        "questions": 800,
        "replicates": 2000,
        "metrics": {
          "accuracy": {
            "estimate": 0.0,
            "ci95": [
              0.0,
              0.0
            ]
          },
          "brier": {
            "estimate": 0.0,
            "ci95": [
              0.0,
              0.0
            ]
          },
          "nll": {
            "estimate": 0.0,
            "ci95": [
              0.0,
              0.0
            ]
          }
        }
      }
    }
  },
  "validation_curves": {
    "17": {
      "reinforce": [
        0.10525166365214286,
        0.12117375417225826,
        0.10056672435834922,
        0.12820932515977224,
        0.11795555333136519,
        0.13667396504524945,
        0.10247942364228006,
        0.1155518692646555,
        0.10779189981201505,
        0.10184562992595193,
        0.09559045068776588,
        0.10363244837789011,
        0.11131992410993118
      ],
      "pathwise": [
        0.10525166365214286,
        0.11065971989234902,
        0.09824757497096048,
        0.10657535849685988,
        0.10125011199781372,
        0.11510530208956354,
        0.1088730146333369,
        0.10679709966254954,
        0.1091377949351295,
        0.10441830290000402,
        0.10627546500851459,
        0.104999833073343,
        0.10231358827323535
      ],
      "supervised_control": [
        0.10525166365214286,
        0.1170103724525552,
        0.09544566093039751,
        0.10298943942544174,
        0.11113089717509701,
        0.10582123057009545,
        0.10669869198125166,
        0.11565393606109643,
        0.10731978267603598,
        0.10381348879637402,
        0.1051709352911545,
        0.10399307047526707,
        0.10163697996214222
      ]
    },
    "42": {
      "reinforce": [
        0.1071913138430586,
        0.10377996122888916,
        0.10310212435470797,
        0.1049493730019816,
        0.1003548910545588,
        0.10705100129739843,
        0.10501917955379572,
        0.10237785883997365,
        0.10095500779253257,
        0.0975886054619946,
        0.09947045925175825,
        0.10244753233010398,
        0.12085995974123508
      ],
      "pathwise": [
        0.1071913138430586,
        0.10240869754659812,
        0.10712373726064847,
        0.11106727240314371,
        0.10569180604129055,
        0.11352077008710619,
        0.10221139976235721,
        0.10494969294374147,
        0.10567522695429225,
        0.10426949485443568,
        0.10381806706524084,
        0.1088538830265569,
        0.12025554920093047
      ],
      "supervised_control": [
        0.1071913138430586,
        0.10093467402595857,
        0.09649431923038009,
        0.0997434978569271,
        0.09846773433976905,
        0.1055279856369447,
        0.09749732962826968,
        0.09925536711833527,
        0.10009254807590924,
        0.0975074993691917,
        0.09881885361931592,
        0.10095188438534601,
        0.10791935218434073
      ]
    },
    "83": {
      "reinforce": [
        0.09896155878389151,
        0.11207710387205463,
        0.10643482040185522,
        0.11008471482831858,
        0.15978935213367723,
        0.11076701087399243,
        0.11080816360842878,
        0.10352241231692563,
        0.10690213270748655,
        0.10033565235208199,
        0.1121528359071667,
        0.09959281776578198,
        0.1002893951963929
      ],
      "pathwise": [
        0.09896155878389151,
        0.10576785811327319,
        0.1119189157873618,
        0.10377035712901829,
        0.101134715685967,
        0.10383180125492152,
        0.10782662054546012,
        0.10116922768142117,
        0.10463641467225038,
        0.10265626522409704,
        0.10178678888427335,
        0.1039241634484645,
        0.10219550067863968
      ],
      "supervised_control": [
        0.09896155878389151,
        0.10096507843476553,
        0.10135653519125114,
        0.10307073141499946,
        0.10203415531357753,
        0.10494437655206518,
        0.10969654279162722,
        0.10307231324468878,
        0.1058754957502558,
        0.10402873280645988,
        0.10363822283071739,
        0.10643811765449535,
        0.10429820518581277
      ]
    }
  },
  "rerun_check": {
    "17": {
      "reinforce": {
        "published_epoch": 10,
        "rerun_epoch": 10,
        "published_test_brier": 0.10353795868889026,
        "rerun_test_brier": 0.10353795868889026,
        "absolute_difference": 0.0,
        "status": "identical_checkpoint"
      },
      "supervised_control": {
        "published_epoch": 2,
        "rerun_epoch": 2,
        "published_test_brier": 0.09887157277906948,
        "rerun_test_brier": 0.09887157277906948,
        "absolute_difference": 0.0,
        "status": "identical_checkpoint"
      }
    },
    "42": {
      "reinforce": {
        "published_epoch": 9,
        "rerun_epoch": 9,
        "published_test_brier": 0.10519841661327942,
        "rerun_test_brier": 0.10519841661327942,
        "absolute_difference": 0.0,
        "status": "identical_checkpoint"
      },
      "supervised_control": {
        "published_epoch": 2,
        "rerun_epoch": 2,
        "published_test_brier": 0.0982134248610902,
        "rerun_test_brier": 0.0982134248610902,
        "absolute_difference": 0.0,
        "status": "identical_checkpoint"
      }
    },
    "83": {
      "reinforce": {
        "published_epoch": 0,
        "rerun_epoch": 0,
        "published_test_brier": 0.10010229722258146,
        "rerun_test_brier": 0.10010229722258146,
        "absolute_difference": 0.0,
        "status": "identical_checkpoint"
      },
      "supervised_control": {
        "published_epoch": 0,
        "rerun_epoch": 0,
        "published_test_brier": 0.10010229722258146,
        "rerun_test_brier": 0.10010229722258146,
        "absolute_difference": 0.0,
        "status": "identical_checkpoint"
      }
    }
  },
  "gradient_diagnostics": {
    "17": {
      "0.5": {
        "reinforce": {
          "trace_variance_per_estimate": 0.46284149843625383,
          "mean_norm": 0.6598389959377321,
          "signal_to_noise": 1.0333015385554893,
          "cosine_with_direct_brier": 0.9842502317484433,
          "share_clipped_at_1": 0.2734375
        },
        "pathwise": {
          "trace_variance_per_estimate": 0.00845221541550655,
          "mean_norm": 0.6454276063316989,
          "signal_to_noise": 85.8120535702609,
          "cosine_with_direct_brier": 0.9905063608346889,
          "share_clipped_at_1": 0.25
        },
        "variance_ratio_reinforce_over_pathwise": 54.75978494195954,
        "agreement_ratio_mean": 0.6462404959492155,
        "agreement_ratio_max": 2.1452463817447716
      },
      "0.2": {
        "reinforce": {
          "trace_variance_per_estimate": 0.4869736717546126,
          "mean_norm": 0.7193704744927283,
          "signal_to_noise": 1.187323023967159,
          "cosine_with_direct_brier": 0.9947243708989945,
          "share_clipped_at_1": 0.26171875
        },
        "pathwise": {
          "trace_variance_per_estimate": 0.0018503159757085983,
          "mean_norm": 0.6971751033506695,
          "signal_to_noise": 476.6724544984792,
          "cosine_with_direct_brier": 0.9995299732998293,
          "share_clipped_at_1": 0.25
        },
        "variance_ratio_reinforce_over_pathwise": 263.18406053221304,
        "agreement_ratio_mean": 0.3919386741168277,
        "agreement_ratio_max": 0.58373299619882
      },
      "0.094": {
        "reinforce": {
          "trace_variance_per_estimate": 0.4063009218850315,
          "mean_norm": 0.6971293234216475,
          "signal_to_noise": 1.2856995562457865,
          "cosine_with_direct_brier": 0.9912163844156276,
          "share_clipped_at_1": 0.24609375
        },
        "pathwise": {
          "trace_variance_per_estimate": 0.0005232238525682875,
          "mean_norm": 0.7062457392591972,
          "signal_to_noise": 1939.8987845674772,
          "cosine_with_direct_brier": 0.9999787942523685,
          "share_clipped_at_1": 0.25
        },
        "variance_ratio_reinforce_over_pathwise": 776.5336383092434,
        "agreement_ratio_mean": 0.6574977694156234,
        "agreement_ratio_max": 0.8380482957796178
      },
      "0.076": {
        "reinforce": {
          "trace_variance_per_estimate": 0.36551461172232724,
          "mean_norm": 0.6674477134250594,
          "signal_to_noise": 1.283015108650251,
          "cosine_with_direct_brier": 0.9870454019371077,
          "share_clipped_at_1": 0.2421875
        },
        "pathwise": {
          "trace_variance_per_estimate": 0.00028707658877636435,
          "mean_norm": 0.7080219181693501,
          "signal_to_noise": 3451.956578238144,
          "cosine_with_direct_brier": 0.9999929331939208,
          "share_clipped_at_1": 0.25
        },
        "variance_ratio_reinforce_over_pathwise": 1273.2303016428377,
        "agreement_ratio_mean": 1.5013428312808506,
        "agreement_ratio_max": 2.603263457853887
      },
      "0.05": {
        "reinforce": {
          "trace_variance_per_estimate": 0.4782464320196254,
          "mean_norm": 0.7189984610166613,
          "signal_to_noise": 1.1929225607573308,
          "cosine_with_direct_brier": 0.996488594859608,
          "share_clipped_at_1": 0.2421875
        },
        "pathwise": {
          "trace_variance_per_estimate": 0.00010644368043091452,
          "mean_norm": 0.7091560970193896,
          "signal_to_noise": 8910.709102980092,
          "cosine_with_direct_brier": 0.9999980663323704,
          "share_clipped_at_1": 0.25
        },
        "variance_ratio_reinforce_over_pathwise": 4492.952799861361,
        "agreement_ratio_mean": 0.29347333854653,
        "agreement_ratio_max": 0.45492176428585107
      }
    },
    "42": {
      "0.5": {
        "reinforce": {
          "trace_variance_per_estimate": 0.2771912973195207,
          "mean_norm": 0.592492117214848,
          "signal_to_noise": 1.9141604026010564,
          "cosine_with_direct_brier": 0.9747485685830328,
          "share_clipped_at_1": 0.19921875
        },
        "pathwise": {
          "trace_variance_per_estimate": 0.0033385613311176666,
          "mean_norm": 0.5834500297148322,
          "signal_to_noise": 142.04900848674697,
          "cosine_with_direct_brier": 0.9892566932956796,
          "share_clipped_at_1": 0.0
        },
        "variance_ratio_reinforce_over_pathwise": 83.02716943849703,
        "agreement_ratio_mean": 0.6783879745896075,
        "agreement_ratio_max": 1.5186668911979924
      },
      "0.2": {
        "reinforce": {
          "trace_variance_per_estimate": 0.2884170785880069,
          "mean_norm": 0.6226456908829614,
          "signal_to_noise": 1.9367242726579605,
          "cosine_with_direct_brier": 0.9940841985439475,
          "share_clipped_at_1": 0.1875
        },
        "pathwise": {
          "trace_variance_per_estimate": 0.0007028900444951943,
          "mean_norm": 0.6222810613493973,
          "signal_to_noise": 687.1329998114431,
          "cosine_with_direct_brier": 0.9995022068419931,
          "share_clipped_at_1": 0.0
        },
        "variance_ratio_reinforce_over_pathwise": 410.3302939724292,
        "agreement_ratio_mean": 0.26361974270296656,
        "agreement_ratio_max": 0.5928757461945089
      },
      "0.094": {
        "reinforce": {
          "trace_variance_per_estimate": 0.25145807594360137,
          "mean_norm": 0.6630858096857781,
          "signal_to_noise": 2.374004057848581,
          "cosine_with_direct_brier": 0.9927662324227045,
          "share_clipped_at_1": 0.234375
        },
        "pathwise": {
          "trace_variance_per_estimate": 0.00016147539780585437,
          "mean_norm": 0.6251221497057325,
          "signal_to_noise": 3278.724070568899,
          "cosine_with_direct_brier": 0.9999844544771266,
          "share_clipped_at_1": 0.0
        },
        "variance_ratio_reinforce_over_pathwise": 1557.253175161304,
        "agreement_ratio_mean": 1.1869134887072454,
        "agreement_ratio_max": 2.6694703713187025
      },
      "0.076": {
        "reinforce": {
          "trace_variance_per_estimate": 0.2478367855231179,
          "mean_norm": 0.6313738281380668,
          "signal_to_noise": 2.635549391461546,
          "cosine_with_direct_brier": 0.9925094055317731,
          "share_clipped_at_1": 0.22265625
        },
        "pathwise": {
          "trace_variance_per_estimate": 0.00010725349941810217,
          "mean_norm": 0.6267993771556946,
          "signal_to_noise": 5298.541434061852,
          "cosine_with_direct_brier": 0.9999383321706993,
          "share_clipped_at_1": 0.0
        },
        "variance_ratio_reinforce_over_pathwise": 2310.7571022646575,
        "agreement_ratio_mean": 0.7507869621645225,
        "agreement_ratio_max": 1.6653046768007829
      },
      "0.05": {
        "reinforce": {
          "trace_variance_per_estimate": 0.2580779727844232,
          "mean_norm": 0.618081486326139,
          "signal_to_noise": 2.135000961425198,
          "cosine_with_direct_brier": 0.9920991519826978,
          "share_clipped_at_1": 0.203125
        },
        "pathwise": {
          "trace_variance_per_estimate": 3.900023669546475e-05,
          "mean_norm": 0.6270679357144233,
          "signal_to_noise": 13268.571913919826,
          "cosine_with_direct_brier": 0.9999984252978076,
          "share_clipped_at_1": 0.0
        },
        "variance_ratio_reinforce_over_pathwise": 6617.343756132498,
        "agreement_ratio_mean": 0.4291527642400812,
        "agreement_ratio_max": 0.47181178504984306
      }
    },
    "83": {
      "0.5": {
        "reinforce": {
          "trace_variance_per_estimate": 0.9379789491702113,
          "mean_norm": 1.0131259212989498,
          "signal_to_noise": 1.2294118280724042,
          "cosine_with_direct_brier": 0.9716701540952857,
          "share_clipped_at_1": 0.46484375
        },
        "pathwise": {
          "trace_variance_per_estimate": 0.021000866994369864,
          "mean_norm": 0.9711555070997098,
          "signal_to_noise": 62.50398083635155,
          "cosine_with_direct_brier": 0.9916613115070596,
          "share_clipped_at_1": 0.30859375
        },
        "variance_ratio_reinforce_over_pathwise": 44.66382028045198,
        "agreement_ratio_mean": 1.0390317528476336,
        "agreement_ratio_max": 3.1108105745265986
      },
      "0.2": {
        "reinforce": {
          "trace_variance_per_estimate": 0.9502030822499791,
          "mean_norm": 1.0477055256799697,
          "signal_to_noise": 1.2896964811368419,
          "cosine_with_direct_brier": 0.989264239600379,
          "share_clipped_at_1": 0.484375
        },
        "pathwise": {
          "trace_variance_per_estimate": 0.0047728485235824375,
          "mean_norm": 1.0074089560205581,
          "signal_to_noise": 305.50640483071186,
          "cosine_with_direct_brier": 0.9996899365099464,
          "share_clipped_at_1": 0.28125
        },
        "variance_ratio_reinforce_over_pathwise": 199.08511186874398,
        "agreement_ratio_mean": 0.5198028422450968,
        "agreement_ratio_max": 1.0144624628629946
      },
      "0.094": {
        "reinforce": {
          "trace_variance_per_estimate": 0.8118193318948566,
          "mean_norm": 1.0178130705439359,
          "signal_to_noise": 1.376644686625821,
          "cosine_with_direct_brier": 0.9859072883477893,
          "share_clipped_at_1": 0.37890625
        },
        "pathwise": {
          "trace_variance_per_estimate": 0.00138759177937646,
          "mean_norm": 1.0092563878962901,
          "signal_to_noise": 1222.740641309625,
          "cosine_with_direct_brier": 0.9999867472502315,
          "share_clipped_at_1": 0.25390625
        },
        "variance_ratio_reinforce_over_pathwise": 585.0563140837161,
        "agreement_ratio_mean": 0.6269024917446663,
        "agreement_ratio_max": 0.831940831600837
      },
      "0.076": {
        "reinforce": {
          "trace_variance_per_estimate": 0.7258680385626702,
          "mean_norm": 0.9482305821135979,
          "signal_to_noise": 1.2840918728099542,
          "cosine_with_direct_brier": 0.9833008012590074,
          "share_clipped_at_1": 0.40625
        },
        "pathwise": {
          "trace_variance_per_estimate": 0.0007954545657276127,
          "mean_norm": 1.0049935327589117,
          "signal_to_noise": 1947.9007735808582,
          "cosine_with_direct_brier": 0.999988541600715,
          "share_clipped_at_1": 0.25
        },
        "variance_ratio_reinforce_over_pathwise": 912.519796650748,
        "agreement_ratio_mean": 1.2886335785701934,
        "agreement_ratio_max": 2.4596769321546135
      },
      "0.05": {
        "reinforce": {
          "trace_variance_per_estimate": 0.9184363455696682,
          "mean_norm": 1.018317408106631,
          "signal_to_noise": 1.2225270228346186,
          "cosine_with_direct_brier": 0.9963402226443532,
          "share_clipped_at_1": 0.3984375
        },
        "pathwise": {
          "trace_variance_per_estimate": 0.0003136667891719711,
          "mean_norm": 1.0067536202615717,
          "signal_to_noise": 4997.685269854763,
          "cosine_with_direct_brier": 0.999998956799107,
          "share_clipped_at_1": 0.25
        },
        "variance_ratio_reinforce_over_pathwise": 2928.0637200839456,
        "agreement_ratio_mean": 0.22823046877984618,
        "agreement_ratio_max": 0.29483305814163796
      }
    }
  },
  "noise_replicates": {
    "reinforce": {
      "per_seed": {
        "17": {
          "draws": [
            {
              "draw": "main",
              "test_brier": 0.10353795868889026,
              "selected_epoch": 10
            },
            {
              "draw": "noise1001",
              "test_brier": 0.09958477971636065,
              "selected_epoch": 2
            },
            {
              "draw": "noise1002",
              "test_brier": 0.0943575117178091,
              "selected_epoch": 9
            },
            {
              "draw": "noise1003",
              "test_brier": 0.10885462190777247,
              "selected_epoch": 4
            }
          ],
          "test_brier": {
            "mean": 0.10158371800770812,
            "sample_std": 0.006134571753614905,
            "values": [
              0.10353795868889026,
              0.09958477971636065,
              0.0943575117178091,
              0.10885462190777247
            ]
          }
        },
        "42": {
          "draws": [
            {
              "draw": "main",
              "test_brier": 0.10519841661327942,
              "selected_epoch": 9
            },
            {
              "draw": "noise1001",
              "test_brier": 0.10397147871503436,
              "selected_epoch": 3
            },
            {
              "draw": "noise1002",
              "test_brier": 0.09861655845798437,
              "selected_epoch": 4
            },
            {
              "draw": "noise1003",
              "test_brier": 0.10681786461917256,
              "selected_epoch": 3
            }
          ],
          "test_brier": {
            "mean": 0.10365107960136766,
            "sample_std": 0.003553019275331776,
            "values": [
              0.10519841661327942,
              0.10397147871503436,
              0.09861655845798437,
              0.10681786461917256
            ]
          }
        },
        "83": {
          "draws": [
            {
              "draw": "main",
              "test_brier": 0.10010229722258146,
              "selected_epoch": 0
            },
            {
              "draw": "noise1001",
              "test_brier": 0.10277142610033113,
              "selected_epoch": 11
            },
            {
              "draw": "noise1002",
              "test_brier": 0.09628757504424384,
              "selected_epoch": 10
            },
            {
              "draw": "noise1003",
              "test_brier": 0.09707360043260937,
              "selected_epoch": 7
            }
          ],
          "test_brier": {
            "mean": 0.09905872469994145,
            "sample_std": 0.002971710173704046,
            "values": [
              0.10010229722258146,
              0.10277142610033113,
              0.09628757504424384,
              0.09707360043260937
            ]
          }
        }
      },
      "pooled_test_brier": {
        "mean": 0.10143117410300574,
        "sample_std": 0.004467895566466421,
        "values": [
          0.10353795868889026,
          0.09958477971636065,
          0.0943575117178091,
          0.10885462190777247,
          0.10519841661327942,
          0.10397147871503436,
          0.09861655845798437,
          0.10681786461917256,
          0.10010229722258146,
          0.10277142610033113,
          0.09628757504424384,
          0.09707360043260937
        ]
      }
    },
    "pathwise": {
      "per_seed": {
        "17": {
          "draws": [
            {
              "draw": "main",
              "test_brier": 0.09962411375196488,
              "selected_epoch": 2
            },
            {
              "draw": "noise1001",
              "test_brier": 0.09961351868543376,
              "selected_epoch": 12
            },
            {
              "draw": "noise1002",
              "test_brier": 0.10099567585955302,
              "selected_epoch": 2
            },
            {
              "draw": "noise1003",
              "test_brier": 0.0962117533291581,
              "selected_epoch": 3
            }
          ],
          "test_brier": {
            "mean": 0.09911126540652744,
            "sample_std": 0.0020390720784932872,
            "values": [
              0.09962411375196488,
              0.09961351868543376,
              0.10099567585955302,
              0.0962117533291581
            ]
          }
        },
        "42": {
          "draws": [
            {
              "draw": "main",
              "test_brier": 0.09848894143794036,
              "selected_epoch": 6
            },
            {
              "draw": "noise1001",
              "test_brier": 0.1000946447392786,
              "selected_epoch": 9
            },
            {
              "draw": "noise1002",
              "test_brier": 0.10004436691942789,
              "selected_epoch": 9
            },
            {
              "draw": "noise1003",
              "test_brier": 0.09858985556282747,
              "selected_epoch": 4
            }
          ],
          "test_brier": {
            "mean": 0.09930445216486856,
            "sample_std": 0.0008846061643562034,
            "values": [
              0.09848894143794036,
              0.1000946447392786,
              0.10004436691942789,
              0.09858985556282747
            ]
          }
        },
        "83": {
          "draws": [
            {
              "draw": "main",
              "test_brier": 0.10010229722258146,
              "selected_epoch": 0
            },
            {
              "draw": "noise1001",
              "test_brier": 0.10010229722258146,
              "selected_epoch": 0
            },
            {
              "draw": "noise1002",
              "test_brier": 0.10010229722258146,
              "selected_epoch": 0
            },
            {
              "draw": "noise1003",
              "test_brier": 0.09929478719071466,
              "selected_epoch": 4
            }
          ],
          "test_brier": {
            "mean": 0.09990041971461476,
            "sample_std": 0.00040375501593339913,
            "values": [
              0.10010229722258146,
              0.10010229722258146,
              0.10010229722258146,
              0.09929478719071466
            ]
          }
        }
      },
      "pooled_test_brier": {
        "mean": 0.09943871242867025,
        "sample_std": 0.0012308078394866614,
        "values": [
          0.09962411375196488,
          0.09961351868543376,
          0.10099567585955302,
          0.0962117533291581,
          0.09848894143794036,
          0.1000946447392786,
          0.10004436691942789,
          0.09858985556282747,
          0.10010229722258146,
          0.10010229722258146,
          0.10010229722258146,
          0.09929478719071466
        ]
      }
    },
    "supervised_control_test_brier": {
      "mean": 0.09906243162091372,
      "sample_std": 0.0009587909170585515,
      "values": [
        0.09887157277906948,
        0.0982134248610902,
        0.10010229722258146
      ]
    }
  }
}