Preserve six final near-greedy RL policies

snapshots/2026-10-31-bounded-actual-reward-gradient-v1/README.md

@@ -0,0 +1,13 @@
+# 有效有界学习头的近贪心直接策略梯度
+
+Git https://github.com/luojiaxuan/cvpr-multimodal-evidence @ 42ffae030632781acbc0e9ee79f9bb932701f8f9；合同docs/setground_bounded_actual_rl_20261031.md；入口code/run_setground_bounded_exact_rl.sbatch。
+
+首轮piB_cost父数据gavinlaw/setground-interactive-grounding@a208bffe08f29b94c6c892240ff262e1973c9f6d、父模型gavinlaw/setground-question-policy@0acbb2c4e465297971a76a1d71a36a838baf5462，前缀snapshots/2026-10-31-bounded-credit-target-loss-v1。温度诊断同数据仓@119136aa73f40195f6ee08c845731b50c4168d0a / snapshots/2026-10-31-bounded-stochastic-behavior-v1。完整输入/公开bank/假想终点/标准化/特征源链见父card，旧数据生成成本继续计入。
+
+三原学习头各匹配immediate/return_to_go，200步/128组精确期望/FP64 Adam1e-4/奖励x20/熵0/固定温度0.03。depth0/1随机，depth>=2固定解析贪心。当前自身Qpi和访问分布给出累计直接梯度；即时臂只分配当前质量变化减成本的信用，不是完整轨迹回报梯度。无Qstar/固定教师拟合。六模型以最终200步为主，原三头不是独立RL采样seed，实际/模型、随机/贪心结果分列；复用heldout仅探索。
+
+奖励边界：真实缓存训练奖励，只由128训练组GT与缓存实答验证环境并计算梯度。one-hot真值权重仅传回报/访问递推，actor公开特征、标准化及策略概率仍用原公开bank，不可将真值或未问实答作为actor输入。dev/heldout不训练。入口第六参数actual_cached_train，默认public_model不能用于复现本条件。匹配公开奖励控制数据gavinlaw/setground-interactive-grounding@b6cda8d1a8a97dd48a9ffa0b1c679613b91fb289、模型gavinlaw/setground-question-policy@d34d08fc7075f8236dd272f30add02589d10cbe1，前缀snapshots/2026-10-31-bounded-exact-policy-gradient-v1。
+
+部署贪心复用train_setground_bounded_credit_factorial.bounded_scores，随机策略复用setground_bounded_policy_gradient.probabilities，温度0.03及固定尾部不可遗漏。公开actor.pt含state_dict/mean/scale/binding。records含contract/summary、每步traces及gradients、checkpoint.pt完整六模型/Adam、greedy/<id>.json独立世界及实际路径、stochastic/<id>.json期望值；真实奖励条件另含reward-boundary.json和stochastic_actual域的非取整期望计数。温度链式导数、首次固定12组头参数方向有限差分、父路径及最终端点已核验。
+
+实测：{"actors": {"seed-101/immediate": {"path": "training/seed-101/immediate/actor.pt", "sha256": "2f9cd8a98d995b9c0ac32e75f4c131a471e9e16a9e30235c5f4b368138760f15"}, "seed-101/return_to_go": {"path": "training/seed-101/return_to_go/actor.pt", "sha256": "a3d1b7308b0f37d53c09daeef0c5c911943ca68d18f1c3b6906e90239d4a0b74"}, "seed-17/immediate": {"path": "training/seed-17/immediate/actor.pt", "sha256": "fea0423fc19dc66bbe8a94f43026fdee82e10ce713d3e748bf1836e60a325fcd"}, "seed-17/return_to_go": {"path": "training/seed-17/return_to_go/actor.pt", "sha256": "6a925ff5df988337c634f8577b0273954a26adccf26b53707f82681d899ee9dd"}, "seed-43/immediate": {"path": "training/seed-43/immediate/actor.pt", "sha256": "5a906da203130e6cc1bbbac55f1a7a4391278956f6505158a05ea62c9a8491ff"}, "seed-43/return_to_go": {"path": "training/seed-43/return_to_go/actor.pt", "sha256": "99120d072875476c7522269c75d0df7e4336e9390ada976f40be92e959ccde87"}}, "binding": {"Qstar_labels": false, "arms": ["immediate", "return_to_go"], "budget": 4, "constraints": "score>=two_step;depth>=2_exact_two_step", "cost": 0.005, "deterministic_tail": "depth>=2", "diagnostic_summary_sha256": "b14919bd82869fafabc79acb013cf7d5cf1b104877c8cc8d0dfed42f31f8fd66", "dtype": "float64", "entropy_coefficient": 0, "finite_check_groups": ["b685ade67233822f1ab911f1e355e1444f0b5e79794ac058404694c113147af9", "990ad044feffdaf89afba8e2b55d7ac10cbd08a21b1a0f21c269505543b8e4ba", "a27a330fecfd806f7014c6360e4c6ef48067911046725bfd90529c240f784a17", "cab2fd949dfd5c8b8fd447dad756540f3bfb783098ee547955c2900c7960e359"], "groups_per_update": 128, "inputs_sha256": "5ce0584f2516f7de904b0ea25c06cda7d32fb97dc6d9bb7d1bd6fc9beb0a1aae", "learning_rate": 0.0001, "parent_seeds": [17, 43, 101], "policy_information": "unchanged_public_features_and_observed_history_only", "primary": "actual_train_return_vs_parent_immediate_and_public_reward_return", "public_reward_reference_sha256": "6e5da3523bbb7198e987fa718909d5d9b921019fdb00a01a6190d5b420022989", "revision": "42ffae030632781acbc0e9ee79f9bb932701f8f9", "reward_scale": 20, "reward_source": "actual_cached_train", "source_summary_sha256": "3907af36c7c26689a8018f0e99e340d29561289947543b8ec4800a78cd14f7ff", "temperature": 0.03, "training_truth_access": true, "updates": 200}, "boundary": "Exact on-policy gradient with actual cached training rewards; truth and cached answers used only for reward/environment verification, never policy inputs. No heldout training. Final greedy deployment is primary; stochastic gains separate. Three pretrained heads, not independent RL sampling seeds; reused heldout exploratory.", "cohorts": {"development": {"actual": {"means": {"immediate": {"correct": 34.0, "queries": 34.0, "utility": 0.7366666666666667}, "parent": {"correct": 34.666666666666664, "queries": 33.666666666666664, "utility": 0.7516666666666666}, "return_to_go": {"correct": 34.0, "queries": 33.666666666666664, "utility": 0.7368518518518519}}, "metrics": {"seed-101/immediate": {"accuracy": 0.7555555555555555, "correct": 34, "groups": 9, "queries": 34, "targets": 45, "utility": 0.7366666666666667}, "seed-101/parent": {"accuracy": 0.7555555555555555, "correct": 34, "groups": 9, "queries": 33, "targets": 45, "utility": 0.7372222222222222}, "seed-101/return_to_go": {"accuracy": 0.7555555555555555, "correct": 34, "groups": 9, "queries": 33, "targets": 45, "utility": 0.7372222222222222}, "seed-17/immediate": {"accuracy": 0.7555555555555555, "correct": 34, "groups": 9, "queries": 34, "targets": 45, "utility": 0.7366666666666667}, "seed-17/parent": {"accuracy": 0.7555555555555555, "correct": 34, "groups": 9, "queries": 34, "targets": 45, "utility": 0.7366666666666667}, "seed-17/return_to_go": {"accuracy": 0.7555555555555555, "correct": 34, "groups": 9, "queries": 35, "targets": 45, "utility": 0.7361111111111112}, "seed-43/immediate": {"accuracy": 0.7555555555555555, "correct": 34, "groups": 9, "queries": 34, "targets": 45, "utility": 0.7366666666666667}, "seed-43/parent": {"accuracy": 0.8, "correct": 36, "groups": 9, "queries": 34, "targets": 45, "utility": 0.7811111111111111}, "seed-43/return_to_go": {"accuracy": 0.7555555555555555, "correct": 34, "groups": 9, "queries": 33, "targets": 45, "utility": 0.7372222222222222}}, "minus_parent": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.044814814814814835, 0.00018518518518521412], "groups": 9, "utility_delta": -0.015000000000000001}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.044444444444444474, 0.0], "groups": 9, "utility_delta": -0.014814814814814824}}, "return_minus_immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0003703703703703913, 0.0007407407407407455], "groups": 9, "utility_delta": 0.00018518518518517713}}, "actual_reward_minus_public_reward": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.13370370370370374, 0.0], "groups": 9, "utility_delta": -0.04481481481481482}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.04444444444444448, 0.0007407407407407209], "groups": 9, "utility_delta": -0.014629629629629647}}, "greedy_model": {"means": {"immediate": {"correct": 3.1994580810527693, "queries": 3.8404761447543048, "utility": 0.6206892354867822}, "parent": {"correct": 3.20161679410239, "queries": 3.8370962874214993, "utility": 0.6211378773833705}, "return_to_go": {"correct": 3.199043318152978, "queries": 3.8350369369285313, "utility": 0.6206334789459529}}, "metrics": {"seed-101/immediate": {"correct": 3.199956172830354, "queries": 3.829620296402135, "utility": 0.6208431330840601}, "seed-101/parent": {"correct": 3.1920728358924886, "queries": 3.7929625237479407, "utility": 0.619449754559758}, "seed-101/return_to_go": {"correct": 3.196606648563739, "queries": 3.7929625237479407, "utility": 0.6203565170940082}, "seed-17/immediate": {"correct": 3.203595836041167, "queries": 3.860423386733352, "utility": 0.6214170502745666}, "seed-17/parent": {"correct": 3.201517877599984, "queries": 3.8476508336597584, "utility": 0.6210653213516981}, "seed-17/return_to_go": {"correct": 3.2085007317401817, "queries": 3.8880329239012292, "utility": 0.6222599817285301}, "seed-43/immediate": {"correct": 3.1948222342867867, "queries": 3.8313847511274273, "utility": 0.6198075231017202}, "seed-43/parent": {"correct": 3.211259668814696, "queries": 3.8706755048568002, "utility": 0.6228985562386552}, "seed-43/return_to_go": {"correct": 3.1920225741550126, "queries": 3.8241153631364235, "utility": 0.6192839380153203}}, "minus_parent": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0028918921171087026, 0.0015430372279905847], "groups": 9, "utility_delta": -0.0004486418965880966}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0024681472741618915, 0.0008743357176044365], "groups": 9, "utility_delta": -0.0005043984374175389}}, "return_minus_immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0029949026837205748, 0.002902276652079572], "groups": 9, "utility_delta": -5.575654082944236e-05}}, "groups": 9, "stochastic_actual": {"means": {"immediate": {"correct": 34.48893825490135, "queries": 34.01230328399752, "utility": 0.7475251260622535}, "parent": {"correct": 34.592517109290604, "queries": 33.541215010646475, "utility": 0.7500885940894321}, "return_to_go": {"correct": 34.25486331148133, "queries": 33.74790546606827, "utility": 0.7424703483295471}}, "metrics": {"seed-101/immediate": {"correct": 34.65628420216731, "queries": 33.97617010825545, "utility": 0.7512639988769094}, "seed-101/parent": {"correct": 34.00021684504862, "queries": 32.8857419799214, "utility": 0.7372905176789017}, "seed-101/return_to_go": {"correct": 34.086351534679636, "queries": 33.00423537433193, "utility": 0.7391387922293631}, "seed-17/immediate": {"correct": 34.80700216185262, "queries": 34.53013426016476, "utility": 0.7543055290077444}, "seed-17/parent": {"correct": 34.08216886277545, "queries": 34.02284762530877, "utility": 0.7384799482698384}, "seed-17/return_to_go": {"correct": 34.177636867652645, "queries": 34.9929486064019, "utility": 0.7400625144998355}, "seed-43/immediate": {"correct": 34.003528400684125, "queries": 33.53060548357236, "utility": 0.737005850302107}, "seed-43/parent": {"correct": 35.69516562004775, "queries": 33.715055426709256, "utility": 0.774495316319556}, "seed-43/return_to_go": {"correct": 34.50060153211171, "queries": 33.24653241747097, "utility": 0.7482097382594429}}, "minus_parent": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.03307149235799873, 0.021246958647671215], "groups": 9, "utility_delta": -0.0025634680271784454}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.03450345739325473, 0.010100490961723583], "groups": 9, "utility_delta": -0.007618245759884913}}, "return_minus_immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.013380992313900546, 0.0013693657798747932], "groups": 9, "utility_delta": -0.005054777732706467}}, "stochastic_actual_minus_parent_greedy": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.03977869131396911, 0.022594365479073328], "groups": 9, "utility_delta": -0.0041415406044129875}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0412808620242552, 0.01086556107493861], "groups": 9, "utility_delta": -0.009196318337119456}}, "stochastic_minus_parent_greedy": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.002175974110389648, 0.0012892068623574092], "groups": 9, "utility_delta": -0.00040544845626454425}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.00247655671344789, 0.0002622077768983277], "groups": 9, "utility_delta": -0.0007971069133971338}}, "stochastic_model": {"means": {"immediate": {"correct": 3.199786104477516, "queries": 3.844958393679459, "utility": 0.6207324289271058}, "parent": {"correct": 3.202147047611829, "queries": 3.8305644103003353, "utility": 0.6212765874708639}, "return_to_go": {"correct": 3.1975834379953856, "queries": 3.8351834258207824, "utility": 0.6203407704699733}}, "metrics": {"seed-101/immediate": {"correct": 3.2021495889113303, "queries": 3.8353161308742787, "utility": 0.6212533371278945}, "seed-101/parent": {"correct": 3.1927194629201425, "queries": 3.793062764799637, "utility": 0.6195785787600304}, "seed-101/return_to_go": {"correct": 3.195061149627243, "queries": 3.795001692932854, "utility": 0.6200372214607843}, "seed-17/immediate": {"correct": 3.204171727782701, "queries": 3.870754486196118, "utility": 0.6214805731255596}, "seed-17/parent": {"correct": 3.2060997489181555, "queries": 3.844572151461892, "utility": 0.6219970890263217}, "seed-17/return_to_go": {"correct": 3.2068468490093913, "queries": 3.88230525120919, "utility": 0.6219578435458324}, "seed-43/immediate": {"correct": 3.1930369967385164, "queries": 3.8288045639679797, "utility": 0.6194633765278633}, "seed-43/parent": {"correct": 3.2076219309971865, "queries": 3.8540583146394787, "utility": 0.6222540946262398}, "seed-43/return_to_go": {"correct": 3.190842315349523, "queries": 3.8282433333203048, "utility": 0.619027246403303}}, "minus_parent": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0024176531586996147, 0.0013213048223902376], "groups": 9, "utility_delta": -0.0005441585437581233}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0033747624650390324, 0.0007990671313614392], "groups": 9, "utility_delta": -0.0009358170008907127}}, "return_minus_immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0030565020785079757, 0.0019404233417017222], "groups": 9, "utility_delta": -0.00039165845713258946}}}, "heldout": {"actual": {"means": {"immediate": {"correct": 81.66666666666667, "queries": 90.33333333333333, "utility": 0.6352666666666665}, "parent": {"correct": 83.0, "queries": 90.66666666666667, "utility": 0.6458666666666666}, "return_to_go": {"correct": 84.33333333333333, "queries": 90.0, "utility": 0.6566666666666666}}, "metrics": {"seed-101/immediate": {"accuracy": 0.648, "correct": 81, "groups": 25, "queries": 91, "targets": 125, "utility": 0.6297999999999999}, "seed-101/parent": {"accuracy": 0.664, "correct": 83, "groups": 25, "queries": 90, "targets": 125, "utility": 0.6459999999999999}, "seed-101/return_to_go": {"accuracy": 0.664, "correct": 83, "groups": 25, "queries": 90, "targets": 125, "utility": 0.6459999999999999}, "seed-17/immediate": {"accuracy": 0.664, "correct": 83, "groups": 25, "queries": 90, "targets": 125, "utility": 0.6459999999999999}, "seed-17/parent": {"accuracy": 0.68, "correct": 85, "groups": 25, "queries": 91, "targets": 125, "utility": 0.6617999999999999}, "seed-17/return_to_go": {"accuracy": 0.688, "correct": 86, "groups": 25, "queries": 90, "targets": 125, "utility": 0.67}, "seed-43/immediate": {"accuracy": 0.648, "correct": 81, "groups": 25, "queries": 90, "targets": 125, "utility": 0.6299999999999999}, "seed-43/parent": {"accuracy": 0.648, "correct": 81, "groups": 25, "queries": 91, "targets": 125, "utility": 0.6297999999999999}, "seed-43/return_to_go": {"accuracy": 0.672, "correct": 84, "groups": 25, "queries": 90, "targets": 125, "utility": 0.6539999999999999}}, "minus_parent": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.03206666666666666, 0.0002666666666666728], "groups": 25, "utility_delta": -0.010599999999999997}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [0.0, 0.029533333333333328], "groups": 25, "utility_delta": 0.010799999999999999}}, "return_minus_immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-6.66666666666682e-05, 0.050733333333333325], "groups": 25, "utility_delta": 0.021399999999999995}}, "actual_reward_minus_public_reward": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.005266666666666669, 0.04533333333333335], "groups": 25, "utility_delta": 0.016000000000000004}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [2.220446049250313e-18, 0.021533333333333328], "groups": 25, "utility_delta": 0.0082}}, "greedy_model": {"means": {"immediate": {"correct": 2.9408334660823745, "queries": 3.654728545990304, "utility": 0.5698930504865235}, "parent": {"correct": 2.941029127408744, "queries": 3.6636735572558994, "utility": 0.5698874576954693}, "return_to_go": {"correct": 2.938863078482638, "queries": 3.656493717003982, "utility": 0.5694901471115078}}, "metrics": {"seed-101/immediate": {"correct": 2.9408102620213996, "queries": 3.636517968707143, "utility": 0.5699794625607443}, "seed-101/parent": {"correct": 2.943094937587532, "queries": 3.638158834305889, "utility": 0.5704281933459769}, "seed-101/return_to_go": {"correct": 2.9370343975847337, "queries": 3.638120163773457, "utility": 0.5692162786980796}, "seed-17/immediate": {"correct": 2.9448822011198716, "queries": 3.6826898361145304, "utility": 0.5705629910434017}, "seed-17/parent": {"correct": 2.9431391089990897, "queries": 3.684883962877671, "utility": 0.5702034019854296}, "seed-17/return_to_go": {"correct": 2.945066815748264, "queries": 3.6736589607215926, "utility": 0.5706450683460449}, "seed-43/immediate": {"correct": 2.936807935105853, "queries": 3.6449778331492397, "utility": 0.5691366978554244}, "seed-43/parent": {"correct": 2.93685333563961, "queries": 3.6679778745841385, "utility": 0.5690307777550013}, "seed-43/return_to_go": {"correct": 2.9344880221149174, "queries": 3.6577020265168967, "utility": 0.5686090942903987}}, "minus_parent": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0016598791572018023, 0.0015989844357108024], "groups": 25, "utility_delta": 5.592791054149782e-06}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0015438793831739858, 0.0007677847688540236], "groups": 25, "utility_delta": -0.00039731058396154096}}, "return_minus_immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0017227277725962058, 0.000885515940273798], "groups": 25, "utility_delta": -0.00040290337501569073}}, "groups": 25, "stochastic_actual": {"means": {"immediate": {"correct": 81.68082579274586, "queries": 90.15271100045845, "utility": 0.6354160641418752}, "parent": {"correct": 82.91582894095232, "queries": 90.68834547381364, "utility": 0.645188962432856}, "return_to_go": {"correct": 84.14121754786792, "queries": 90.01578623407505, "utility": 0.6551265831361284}}, "metrics": {"seed-101/immediate": {"correct": 81.0345558165265, "queries": 90.66260097490391, "utility": 0.6301439263372313}, "seed-101/parent": {"correct": 83.24624188516717, "queries": 89.78299594546134, "utility": 0.6480133358922451}, "seed-101/return_to_go": {"correct": 83.17521896947008, "queries": 89.97607887124973, "utility": 0.6474065359815105}, "seed-17/immediate": {"correct": 83.05180076190693, "queries": 90.01873911142332, "utility": 0.6464106582729707}, "seed-17/parent": {"correct": 84.59907395512097, "queries": 91.30646733327133, "utility": 0.6585312981743136}, "seed-17/return_to_go": {"correct": 85.45988237111801, "queries": 90.19273765363903, "utility": 0.6656405114382163}, "seed-43/immediate": {"correct": 80.95612079980418, "queries": 89.77679291504816, "utility": 0.6296936078154239}, "seed-43/parent": {"correct": 80.90217098256886, "queries": 90.97557314270827, "utility": 0.6290222532320091}, "seed-43/return_to_go": {"correct": 83.78855130301568, "queries": 89.87854217733641, "utility": 0.6523327019886581}}, "minus_parent": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.02981959201124532, 0.0016734951305720286], "groups": 25, "utility_delta": -0.009772898290980669}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [0.00037511999756435245, 0.02517984177714417], "groups": 25, "utility_delta": 0.009937620703272339}}, "return_minus_immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [7.454982140403612e-05, 0.046273350338106915], "groups": 25, "utility_delta": 0.019710518994253006}}, "stochastic_actual_minus_parent_greedy": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.03166052645775807, 0.00034786653549000277], "groups": 25, "utility_delta": -0.010450602524791339}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0023019848768010887, 0.02879177571606966], "groups": 25, "utility_delta": 0.009259916469461667}}, "stochastic_minus_parent_greedy": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0017470877081879562, 0.0010816230724823653], "groups": 25, "utility_delta": -0.0002833353738642841}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0011724807167730784, 0.0007556899920391379], "groups": 25, "utility_delta": -0.00021778153769930864}}, "stochastic_model": {"means": {"immediate": {"correct": 2.939430294927929, "queries": 3.6563873327961516, "utility": 0.569604122321605}, "parent": {"correct": 2.940013942499553, "queries": 3.6621029891455463, "utility": 0.5696922735541831}, "return_to_go": {"correct": 2.939891094944701, "queries": 3.6617085662340596, "utility": 0.5696696761577699}}, "metrics": {"seed-101/immediate": {"correct": 2.9387649335245825, "queries": 3.641046009145208, "utility": 0.5695477566591904}, "seed-101/parent": {"correct": 2.940825875955773, "queries": 3.6440345978702817, "utility": 0.5699450022018032}, "seed-101/return_to_go": {"correct": 2.9377569894391593, "queries": 3.6505176195551985, "utility": 0.5692988097900558}, "seed-17/immediate": {"correct": 2.9428268406844365, "queries": 3.681543306008119, "utility": 0.5701576516068467}, "seed-17/parent": {"correct": 2.942639220527344, "queries": 3.6828252519610967, "utility": 0.5701137178456633}, "seed-17/return_to_go": {"correct": 2.946513994598226, "queries": 3.678672417615936, "utility": 0.5709094368315655}, "seed-43/immediate": {"correct": 2.936699110574767, "queries": 3.6465726832351275, "utility": 0.5691069586987778}, "seed-43/parent": {"correct": 2.9365767310155437, "queries": 3.659449117605259, "utility": 0.5690181006150826}, "seed-43/return_to_go": {"correct": 2.9354023007967185, "queries": 3.6559356615310445, "utility": 0.5688007818516885}}, "minus_parent": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.001414641033932462, 0.0011088459142421857], "groups": 25, "utility_delta": -8.815123257803092e-05}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0008063464614831288, 0.0008713766418447429], "groups": 25, "utility_delta": -2.259739641305547e-05}}, "return_minus_immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0009619404599025019, 0.001126577477026933], "groups": 25, "utility_delta": 6.555383616497545e-05}}}, "train": {"actual": {"means": {"immediate": {"correct": 444.3333333333333, "queries": 456.0, "utility": 0.6764583333333333}, "parent": {"correct": 443.6666666666667, "queries": 459.6666666666667, "utility": 0.6752734375}, "return_to_go": {"correct": 456.3333333333333, "queries": 456.3333333333333, "utility": 0.6951953125000001}}, "metrics": {"seed-101/immediate": {"accuracy": 0.696875, "correct": 446, "groups": 128, "queries": 455, "targets": 640, "utility": 0.6791015625000001}, "seed-101/parent": {"accuracy": 0.6953125, "correct": 445, "groups": 128, "queries": 458, "targets": 640, "utility": 0.6774218750000001}, "seed-101/return_to_go": {"accuracy": 0.7109375, "correct": 455, "groups": 128, "queries": 455, "targets": 640, "utility": 0.6931640625000001}, "seed-17/immediate": {"accuracy": 0.69375, "correct": 444, "groups": 128, "queries": 455, "targets": 640, "utility": 0.6759765625}, "seed-17/parent": {"accuracy": 0.6921875, "correct": 443, "groups": 128, "queries": 457, "targets": 640, "utility": 0.6743359375}, "seed-17/return_to_go": {"accuracy": 0.7203125, "correct": 461, "groups": 128, "queries": 457, "targets": 640, "utility": 0.7024609375000002}, "seed-43/immediate": {"accuracy": 0.6921875, "correct": 443, "groups": 128, "queries": 458, "targets": 640, "utility": 0.674296875}, "seed-43/parent": {"accuracy": 0.6921875, "correct": 443, "groups": 128, "queries": 464, "targets": 640, "utility": 0.6740625}, "seed-43/return_to_go": {"accuracy": 0.7078125, "correct": 453, "groups": 128, "queries": 457, "targets": 640, "utility": 0.6899609375}}, "minus_parent": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0060416666666666605, 0.008086588541666618], "groups": 128, "utility_delta": 0.0011848958333333355}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [0.01002604166666667, 0.03140625], "groups": 128, "utility_delta": 0.019921875000000002}}, "return_minus_immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [0.00828125, 0.030794270833333325], "groups": 128, "utility_delta": 0.018736979166666667}}, "actual_reward_minus_public_reward": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.017460937499999996, 0.01881510416666666], "groups": 128, "utility_delta": 0.0001171875000000001}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [0.009466145833333333, 0.033463541666666666], "groups": 128, "utility_delta": 0.020416666666666666}}, "greedy_model": {"means": {"immediate": {"correct": 3.042988419697964, "queries": 3.595654921237729, "utility": 0.5906194093334043}, "parent": {"correct": 3.0427394529200242, "queries": 3.600841643908403, "utility": 0.5905436823644628}, "return_to_go": {"correct": 3.0402367442366436, "queries": 3.5949607252923435, "utility": 0.5900725452208669}}, "metrics": {"seed-101/immediate": {"correct": 3.0454307892943655, "queries": 3.599870989950534, "utility": 0.5910868029091204}, "seed-101/parent": {"correct": 3.044182938476644, "queries": 3.60904248252184, "utility": 0.5907913752827196}, "seed-101/return_to_go": {"correct": 3.0412428571217056, "queries": 3.5965988465704144, "utility": 0.590265577191489}, "seed-17/immediate": {"correct": 3.0421499664430627, "queries": 3.585758895022402, "utility": 0.5905011988135006}, "seed-17/parent": {"correct": 3.041469094131856, "queries": 3.5880424143931946, "utility": 0.5903536067544052}, "seed-17/return_to_go": {"correct": 3.039348182219145, "queries": 3.6029034587548514, "utility": 0.5898551191500547}, "seed-43/immediate": {"correct": 3.0413845033564644, "queries": 3.6013348787402504, "utility": 0.5902702262775917}, "seed-43/parent": {"correct": 3.0425663261515714, "queries": 3.605440034810175, "utility": 0.5904860650562633}, "seed-43/return_to_go": {"correct": 3.04011919336908, "queries": 3.5853798705517645, "utility": 0.5900969393210571}}, "minus_parent": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.00048751426323919263, 0.0006329775255394162], "groups": 128, "utility_delta": 7.57269689415324e-05}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0009085572131089318, -3.006855628431709e-05], "groups": 128, "utility_delta": -0.00047113714359576023}}, "return_minus_immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0010801755849241994, -2.3011360052362667e-05], "groups": 128, "utility_delta": -0.0005468641125372926}}, "groups": 128, "stochastic_actual": {"means": {"immediate": {"correct": 443.6067737542847, "queries": 457.28034702451606, "utility": 0.6752730704354248}, "parent": {"correct": 443.78451748688485, "queries": 460.5681253392551, "utility": 0.6754223661771929}, "return_to_go": {"correct": 455.73370229447863, "queries": 457.7369643086611, "utility": 0.6942035596668156}}, "metrics": {"seed-101/immediate": {"correct": 443.68108953534704, "queries": 456.9344645750789, "utility": 0.6754026998765161}, "seed-101/parent": {"correct": 444.9026172068928, "queries": 458.7169461216791, "utility": 0.6772417086778918}, "seed-101/return_to_go": {"correct": 454.5277685518736, "queries": 457.2279992362369, "utility": 0.6923391696421368}, "seed-17/immediate": {"correct": 443.642918166812, "queries": 455.8295893102112, "utility": 0.6753862163032136}, "seed-17/parent": {"correct": 443.983530535868, "queries": 458.61416841167284, "utility": 0.6758096505087128}, "seed-17/return_to_go": {"correct": 459.76647796548224, "queries": 458.0123380687979, "utility": 0.7004940148652534}, "seed-43/immediate": {"correct": 443.49631356069494, "queries": 459.0769871882581, "utility": 0.6750302951265448}, "seed-43/parent": {"correct": 442.46740471789377, "queries": 464.3732614844133, "utility": 0.6732157393449743}, "seed-43/return_to_go": {"correct": 452.9068603660799, "queries": 457.9705556209485, "utility": 0.6897774944930567}}, "minus_parent": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.006771931548882401, 0.006083532253312313], "groups": 128, "utility_delta": -0.00014929574176811245}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [0.010177426990145054, 0.028685280936241998], "groups": 128, "utility_delta": 0.018781193489622754}}, "return_minus_immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [0.008880674165983518, 0.030687709243249518], "groups": 128, "utility_delta": 0.018930489231390867}}, "stochastic_actual_minus_parent_greedy": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.007168652416483481, 0.006605229055693285], "groups": 128, "utility_delta": -3.670645751749921e-07}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [0.009555878758898385, 0.03003114487723645], "groups": 128, "utility_delta": 0.018930122166815694}}, "stochastic_minus_parent_greedy": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0008007780678030937, 0.00030806406530251246], "groups": 128, "utility_delta": -0.00023753642742647485}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0011032619914723995, -0.00010857088185279199], "groups": 128, "utility_delta": -0.0005909029848051727}}, "stochastic_model": {"means": {"immediate": {"correct": 3.04152532589781, "queries": 3.5997838485051417, "utility": 0.5903061459370362}, "parent": {"correct": 3.041575395620372, "queries": 3.6047206649903902, "utility": 0.5902914757991226}, "return_to_go": {"correct": 3.0397670853249914, "queries": 3.6001275370681505, "utility": 0.5899527793796576}}, "metrics": {"seed-101/immediate": {"correct": 3.043162609180934, "queries": 3.604909954714109, "utility": 0.5906079720626162}, "seed-101/parent": {"correct": 3.042853977004024, "queries": 3.6099713511148965, "utility": 0.5905209386452304}, "seed-101/return_to_go": {"correct": 3.0409134760958865, "queries": 3.6029916556066595, "utility": 0.590167736941144}, "seed-17/immediate": {"correct": 3.0411078754269516, "queries": 3.589986282215186, "utility": 0.5902716436743145}, "seed-17/parent": {"correct": 3.0399779229080472, "queries": 3.5931719497311088, "utility": 0.5900297248329539}, "seed-17/return_to_go": {"correct": 3.037689331366563, "queries": 3.6053961721490753, "utility": 0.5895108854125672}, "seed-43/immediate": {"correct": 3.0403054930855444, "queries": 3.6044553085861293, "utility": 0.5900388220741781}, "seed-43/parent": {"correct": 3.0418942869490455, "queries": 3.611018694125165, "utility": 0.5903237639191833}, "seed-43/return_to_go": {"correct": 3.0406984485125252, "queries": 3.591994783448717, "utility": 0.5901797157852615}}, "minus_parent": {"immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.00045479502594299485, 0.00048005602221577563], "groups": 128, "utility_delta": 1.467013791376032e-05}, "return_to_go": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0007174059463525752, 3.637135775667251e-05], "groups": 128, "utility_delta": -0.0003386964194649375}}, "return_minus_immediate": {"bootstrap_replicates": 20000, "bootstrap_seed": 61031, "group_bootstrap_95": [-0.0008000406852466363, 0.00010637352827616666], "groups": 128, "utility_delta": -0.0003533665573786978}}}}, "gpus": 0, "new_vlm_calls": 0, "primary_pass": true, "sampled_episodes": 0, "status": "COMPLETE_BOUNDED_EXACT_PAIRED_POLICY_GRADIENT", "verification": {"actual_paths": 1782, "evaluation_cached_answer_checks": 1852, "final_greedy_world_replays": 213840, "first_parameter_finite_difference_errors": [3.578566254874261e-10, 2.6118136750574728e-09, 9.013187609457907e-10, 5.551115123125783e-10, 2.3816063527233868e-09, 1.32187979160793e-09, 1.010971941578509e-09, 5.14038680532795e-15, 1.468800196793222e-09, 4.994137360603279e-10, 4.322114803307909e-09, 2.547491572157464e-15], "legal_action_gradient_passes": 1318090800, "maximum_final_error": 7.105427357601002e-15, "maximum_training_score_error": 1.0658141036401503e-14, "optimizer_updates": 1200, "parent_greedy_world_replays": 58320, "score_checks": 1320548400, "state_gradient_passes": 140250000, "training_reward_cached_answer_checks": 1449}, "wall_seconds": 395.7789507830021}

snapshots/2026-10-31-bounded-actual-reward-gradient-v1/seed-101/immediate/actor.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f9cd8a98d995b9c0ac32e75f4c131a471e9e16a9e30235c5f4b368138760f15
+size 48452

snapshots/2026-10-31-bounded-actual-reward-gradient-v1/seed-101/return_to_go/actor.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3d1b7308b0f37d53c09daeef0c5c911943ca68d18f1c3b6906e90239d4a0b74
+size 48452

snapshots/2026-10-31-bounded-actual-reward-gradient-v1/seed-17/immediate/actor.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fea0423fc19dc66bbe8a94f43026fdee82e10ce713d3e748bf1836e60a325fcd
+size 48452

snapshots/2026-10-31-bounded-actual-reward-gradient-v1/seed-17/return_to_go/actor.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a925ff5df988337c634f8577b0273954a26adccf26b53707f82681d899ee9dd
+size 48452

snapshots/2026-10-31-bounded-actual-reward-gradient-v1/seed-43/immediate/actor.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a906da203130e6cc1bbbac55f1a7a4391278956f6505158a05ea62c9a8491ff
+size 48452

snapshots/2026-10-31-bounded-actual-reward-gradient-v1/seed-43/return_to_go/actor.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99120d072875476c7522269c75d0df7e4336e9390ada976f40be92e959ccde87
+size 48452