Auto-sync: 2026-06-30 15:00:21 (part 3)
Browse files- results/paper_analysis.json +197 -3
- results/paper_analysis.md +4 -2
- results/paper_table_status.json +55 -17
- results/paper_table_status.md +4 -2
- scripts/build_paper_analysis.py +17 -1
- scripts/build_paper_table_status.py +22 -2
- scripts/eval_maniskill_policy_rollout.py +46 -0
- scripts/slurm/eval_maniskill_policy_rollout.sbatch +4 -0
- scripts/slurm/summarize_h16_policy_ckpt.sbatch +27 -1
results/paper_analysis.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"best_clean_key": "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001",
|
| 3 |
-
"generated_utc": "2026-06-
|
| 4 |
"mechanism_gap": {
|
| 5 |
"best_clean_vs_direct_same_ckpt": 0.0776811594202898,
|
| 6 |
"best_clean_vs_h16": 0.06318840579710144,
|
|
@@ -1917,6 +1917,11 @@
|
|
| 1917 |
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_scales035040_summary.json",
|
| 1918 |
"std_success": 0.013130146843949137
|
| 1919 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1920 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger002": {
|
| 1921 |
"ci95_success": 0.03115620450679224,
|
| 1922 |
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.02",
|
|
@@ -2111,6 +2116,103 @@
|
|
| 2111 |
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p03_summary.json",
|
| 2112 |
"std_success": 0.012819330079707819
|
| 2113 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2114 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001": {
|
| 2115 |
"ci95_success": 0.028111505659259583,
|
| 2116 |
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01",
|
|
@@ -2209,9 +2311,101 @@
|
|
| 2209 |
"std_success": 0.011315490606343858
|
| 2210 |
},
|
| 2211 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull": {
|
|
|
|
| 2212 |
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only",
|
| 2213 |
-
"
|
| 2214 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2215 |
},
|
| 2216 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8": {
|
| 2217 |
"candidate_oracle_rollouts": 8,
|
|
|
|
| 1 |
{
|
| 2 |
"best_clean_key": "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001",
|
| 3 |
+
"generated_utc": "2026-06-30T19:13:18+00:00",
|
| 4 |
"mechanism_gap": {
|
| 5 |
"best_clean_vs_direct_same_ckpt": 0.0776811594202898,
|
| 6 |
"best_clean_vs_h16": 0.06318840579710144,
|
|
|
|
| 1917 |
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_scales035040_summary.json",
|
| 1918 |
"std_success": 0.013130146843949137
|
| 1919 |
},
|
| 1920 |
+
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_stacknowg": {
|
| 1921 |
+
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, no wrong-gripper component on Stack",
|
| 1922 |
+
"missing": true,
|
| 1923 |
+
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_stacknowg_summary.json"
|
| 1924 |
+
},
|
| 1925 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger002": {
|
| 1926 |
"ci95_success": 0.03115620450679224,
|
| 1927 |
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.02",
|
|
|
|
| 2116 |
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p03_summary.json",
|
| 2117 |
"std_success": 0.012819330079707819
|
| 2118 |
},
|
| 2119 |
+
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001": {
|
| 2120 |
+
"ci95_success": 0.03920403094295293,
|
| 2121 |
+
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01",
|
| 2122 |
+
"mean_action_mse_to_best": 0.41923150534080206,
|
| 2123 |
+
"mean_progress": 0.574088785921579,
|
| 2124 |
+
"mean_success": 0.36057971014492757,
|
| 2125 |
+
"num_completed": 3,
|
| 2126 |
+
"per_task_success": {
|
| 2127 |
+
"LiftPegUpright-v1": {
|
| 2128 |
+
"mean_num_groups": 102.0,
|
| 2129 |
+
"mean_success": 0.26340865087329823,
|
| 2130 |
+
"std_success": 0.06057881385012102
|
| 2131 |
+
},
|
| 2132 |
+
"PickCube-v1": {
|
| 2133 |
+
"mean_num_groups": 196.66666666666666,
|
| 2134 |
+
"mean_success": 0.3305568224046485,
|
| 2135 |
+
"std_success": 0.04196214617114575
|
| 2136 |
+
},
|
| 2137 |
+
"PullCube-v1": {
|
| 2138 |
+
"mean_num_groups": 81.0,
|
| 2139 |
+
"mean_success": 0.228271436166173,
|
| 2140 |
+
"std_success": 0.03525140677829959
|
| 2141 |
+
},
|
| 2142 |
+
"PushCube-v1": {
|
| 2143 |
+
"mean_num_groups": 102.0,
|
| 2144 |
+
"mean_success": 0.7604066042026859,
|
| 2145 |
+
"std_success": 0.08151850231627764
|
| 2146 |
+
},
|
| 2147 |
+
"StackCube-v1": {
|
| 2148 |
+
"mean_num_groups": 93.33333333333333,
|
| 2149 |
+
"mean_success": 0.19707329707329704,
|
| 2150 |
+
"std_success": 0.0831467196651622
|
| 2151 |
+
}
|
| 2152 |
+
},
|
| 2153 |
+
"seed_action_mse_to_best": {
|
| 2154 |
+
"0": 0.39629442195691494,
|
| 2155 |
+
"1": 0.41279180422911177,
|
| 2156 |
+
"2": 0.4486082898363795
|
| 2157 |
+
},
|
| 2158 |
+
"seed_progress": {
|
| 2159 |
+
"0": 0.559952266274722,
|
| 2160 |
+
"1": 0.5719372708739145,
|
| 2161 |
+
"2": 0.5903768206161002
|
| 2162 |
+
},
|
| 2163 |
+
"seed_success": {
|
| 2164 |
+
"0": 0.3582608695652174,
|
| 2165 |
+
"1": 0.34608695652173915,
|
| 2166 |
+
"2": 0.3773913043478261
|
| 2167 |
+
},
|
| 2168 |
+
"selected_candidate_type_counts": {
|
| 2169 |
+
"retrieval_residual_policy_residual": 1444,
|
| 2170 |
+
"retrieval_residual_residual_near_miss": 91,
|
| 2171 |
+
"retrieval_residual_residual_near_miss+residual_wrong_gripper": 5,
|
| 2172 |
+
"retrieval_residual_residual_no_op": 38,
|
| 2173 |
+
"retrieval_residual_residual_no_op+residual_wrong_gripper": 27,
|
| 2174 |
+
"retrieval_residual_residual_wrong_gripper": 120
|
| 2175 |
+
},
|
| 2176 |
+
"selected_residual_scale_counts": {
|
| 2177 |
+
"0.35": 1506,
|
| 2178 |
+
"0.4": 36,
|
| 2179 |
+
"0.45": 183
|
| 2180 |
+
},
|
| 2181 |
+
"selected_type_outcomes": {
|
| 2182 |
+
"retrieval_residual_policy_residual": {
|
| 2183 |
+
"count": 1444.0,
|
| 2184 |
+
"mean_progress": 0.5575818072795194,
|
| 2185 |
+
"success_rate": 0.3337950138504155
|
| 2186 |
+
},
|
| 2187 |
+
"retrieval_residual_residual_near_miss": {
|
| 2188 |
+
"count": 91.0,
|
| 2189 |
+
"mean_progress": 0.5661385127926594,
|
| 2190 |
+
"success_rate": 0.38461538461538464
|
| 2191 |
+
},
|
| 2192 |
+
"retrieval_residual_residual_near_miss+residual_wrong_gripper": {
|
| 2193 |
+
"count": 5.0,
|
| 2194 |
+
"mean_progress": 0.7557853102684021,
|
| 2195 |
+
"success_rate": 0.6
|
| 2196 |
+
},
|
| 2197 |
+
"retrieval_residual_residual_no_op": {
|
| 2198 |
+
"count": 38.0,
|
| 2199 |
+
"mean_progress": 0.7757954887653652,
|
| 2200 |
+
"success_rate": 0.5263157894736842
|
| 2201 |
+
},
|
| 2202 |
+
"retrieval_residual_residual_no_op+residual_wrong_gripper": {
|
| 2203 |
+
"count": 27.0,
|
| 2204 |
+
"mean_progress": 0.6916872307106301,
|
| 2205 |
+
"success_rate": 0.48148148148148145
|
| 2206 |
+
},
|
| 2207 |
+
"retrieval_residual_residual_wrong_gripper": {
|
| 2208 |
+
"count": 120.0,
|
| 2209 |
+
"mean_progress": 0.6808475915446053,
|
| 2210 |
+
"success_rate": 0.575
|
| 2211 |
+
}
|
| 2212 |
+
},
|
| 2213 |
+
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_challenger0p01_norm_summary.json",
|
| 2214 |
+
"std_success": 0.015780472566743406
|
| 2215 |
+
},
|
| 2216 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001": {
|
| 2217 |
"ci95_success": 0.028111505659259583,
|
| 2218 |
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01",
|
|
|
|
| 2311 |
"std_success": 0.011315490606343858
|
| 2312 |
},
|
| 2313 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull": {
|
| 2314 |
+
"ci95_success": 0.04121578446082103,
|
| 2315 |
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only",
|
| 2316 |
+
"mean_action_mse_to_best": 0.41794628433532255,
|
| 2317 |
+
"mean_progress": 0.5735963408033935,
|
| 2318 |
+
"mean_success": 0.36000000000000004,
|
| 2319 |
+
"num_completed": 3,
|
| 2320 |
+
"per_task_success": {
|
| 2321 |
+
"LiftPegUpright-v1": {
|
| 2322 |
+
"mean_num_groups": 102.0,
|
| 2323 |
+
"mean_success": 0.26340865087329823,
|
| 2324 |
+
"std_success": 0.06057881385012102
|
| 2325 |
+
},
|
| 2326 |
+
"PickCube-v1": {
|
| 2327 |
+
"mean_num_groups": 196.66666666666666,
|
| 2328 |
+
"mean_success": 0.3305568224046485,
|
| 2329 |
+
"std_success": 0.04196214617114575
|
| 2330 |
+
},
|
| 2331 |
+
"PullCube-v1": {
|
| 2332 |
+
"mean_num_groups": 81.0,
|
| 2333 |
+
"mean_success": 0.228271436166173,
|
| 2334 |
+
"std_success": 0.03525140677829959
|
| 2335 |
+
},
|
| 2336 |
+
"PushCube-v1": {
|
| 2337 |
+
"mean_num_groups": 102.0,
|
| 2338 |
+
"mean_success": 0.757403601199683,
|
| 2339 |
+
"std_success": 0.07781968814460603
|
| 2340 |
+
},
|
| 2341 |
+
"StackCube-v1": {
|
| 2342 |
+
"mean_num_groups": 93.33333333333333,
|
| 2343 |
+
"mean_success": 0.19707329707329704,
|
| 2344 |
+
"std_success": 0.0831467196651622
|
| 2345 |
+
}
|
| 2346 |
+
},
|
| 2347 |
+
"seed_action_mse_to_best": {
|
| 2348 |
+
"0": 0.3939786813845453,
|
| 2349 |
+
"1": 0.41151866878018434,
|
| 2350 |
+
"2": 0.4483415028412381
|
| 2351 |
+
},
|
| 2352 |
+
"seed_progress": {
|
| 2353 |
+
"0": 0.559967816375007,
|
| 2354 |
+
"1": 0.5704443854190733,
|
| 2355 |
+
"2": 0.5903768206161002
|
| 2356 |
+
},
|
| 2357 |
+
"seed_success": {
|
| 2358 |
+
"0": 0.3582608695652174,
|
| 2359 |
+
"1": 0.3443478260869565,
|
| 2360 |
+
"2": 0.3773913043478261
|
| 2361 |
+
},
|
| 2362 |
+
"selected_candidate_type_counts": {
|
| 2363 |
+
"retrieval_residual_policy_residual": 1463,
|
| 2364 |
+
"retrieval_residual_residual_near_miss": 72,
|
| 2365 |
+
"retrieval_residual_residual_near_miss+residual_wrong_gripper": 5,
|
| 2366 |
+
"retrieval_residual_residual_no_op": 38,
|
| 2367 |
+
"retrieval_residual_residual_no_op+residual_wrong_gripper": 27,
|
| 2368 |
+
"retrieval_residual_residual_wrong_gripper": 120
|
| 2369 |
+
},
|
| 2370 |
+
"selected_residual_scale_counts": {
|
| 2371 |
+
"0.35": 1516,
|
| 2372 |
+
"0.4": 35,
|
| 2373 |
+
"0.45": 174
|
| 2374 |
+
},
|
| 2375 |
+
"selected_type_outcomes": {
|
| 2376 |
+
"retrieval_residual_policy_residual": {
|
| 2377 |
+
"count": 1463.0,
|
| 2378 |
+
"mean_progress": 0.5617103827593661,
|
| 2379 |
+
"success_rate": 0.3390293916609706
|
| 2380 |
+
},
|
| 2381 |
+
"retrieval_residual_residual_near_miss": {
|
| 2382 |
+
"count": 72.0,
|
| 2383 |
+
"mean_progress": 0.47270800791577333,
|
| 2384 |
+
"success_rate": 0.2777777777777778
|
| 2385 |
+
},
|
| 2386 |
+
"retrieval_residual_residual_near_miss+residual_wrong_gripper": {
|
| 2387 |
+
"count": 5.0,
|
| 2388 |
+
"mean_progress": 0.7557853102684021,
|
| 2389 |
+
"success_rate": 0.6
|
| 2390 |
+
},
|
| 2391 |
+
"retrieval_residual_residual_no_op": {
|
| 2392 |
+
"count": 38.0,
|
| 2393 |
+
"mean_progress": 0.7757954887653652,
|
| 2394 |
+
"success_rate": 0.5263157894736842
|
| 2395 |
+
},
|
| 2396 |
+
"retrieval_residual_residual_no_op+residual_wrong_gripper": {
|
| 2397 |
+
"count": 27.0,
|
| 2398 |
+
"mean_progress": 0.6916872307106301,
|
| 2399 |
+
"success_rate": 0.48148148148148145
|
| 2400 |
+
},
|
| 2401 |
+
"retrieval_residual_residual_wrong_gripper": {
|
| 2402 |
+
"count": 120.0,
|
| 2403 |
+
"mean_progress": 0.6808475915446053,
|
| 2404 |
+
"success_rate": 0.575
|
| 2405 |
+
}
|
| 2406 |
+
},
|
| 2407 |
+
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgchallenger0p01_pickpull_norm_summary.json",
|
| 2408 |
+
"std_success": 0.01659024698116427
|
| 2409 |
},
|
| 2410 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8": {
|
| 2411 |
"candidate_oracle_rollouts": 8,
|
results/paper_analysis.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
# Paper Analysis
|
| 2 |
|
| 3 |
-
Generated: `2026-06-
|
| 4 |
|
| 5 |
## Main Seed Statistics
|
| 6 |
|
|
@@ -54,13 +54,15 @@ Generated: `2026-06-30T18:49:56+00:00`
|
|
| 54 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace | K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 branch trace | 3 | 35.65% +/- 1.22 | +/- 3.02 | 57.16% | 0.406 | +5.91 pp |
|
| 55 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger002 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.02 | 3 | 36.00% +/- 1.25 | +/- 3.12 | 57.42% | 0.407 | +6.26 pp |
|
| 56 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01 | 3 | 36.06% +/- 1.41 | +/- 3.49 | 57.38% | 0.407 | +6.32 pp |
|
|
|
|
| 57 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0005 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.005 | 3 | 36.00% +/- 1.42 | +/- 3.54 | 57.34% | 0.407 | +6.26 pp |
|
| 58 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0015 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.015 | 3 | 36.00% +/- 1.25 | +/- 3.12 | 57.38% | 0.407 | +6.26 pp |
|
| 59 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_scale035 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, scale-gated 0.35 | 3 | 36.00% +/- 1.14 | +/- 2.83 | 57.33% | 0.407 | +6.26 pp |
|
| 60 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_scales035040 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, scale-gated 0.35/0.40 | 3 | 36.00% +/- 1.31 | +/- 3.26 | 57.37% | 0.407 | +6.26 pp |
|
| 61 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger003 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.03 | 3 | 35.94% +/- 1.28 | +/- 3.18 | 57.36% | 0.407 | +6.20 pp |
|
| 62 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01 | 3 | 35.94% +/- 1.13 | +/- 2.81 | 57.40% | 0.424 | +6.20 pp |
|
| 63 |
-
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull | K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only |
|
|
|
|
| 64 |
| typed_proposal_lattice_types6_prepend_margin000 | Typed proposal lattice head, six families, policy-prepended margin 0.00 | 3 | 31.30% +/- 3.81 | +/- 9.48 | 54.03% | 0.480 | +1.57 pp |
|
| 65 |
| typed_proposal_lattice_types6_prepend_margin005 | Typed proposal lattice head, six families, policy-prepended margin 0.05 | 3 | 32.17% +/- 3.05 | +/- 7.57 | 54.74% | 0.467 | +2.43 pp |
|
| 66 |
| typed_proposal_lattice_types4safe_prepend_margin005 | Typed proposal lattice head, safe four families, policy-prepended margin 0.05 | 3 | 33.45% +/- 3.39 | +/- 8.43 | 55.47% | 0.456 | +3.71 pp |
|
|
|
|
| 1 |
# Paper Analysis
|
| 2 |
|
| 3 |
+
Generated: `2026-06-30T19:13:18+00:00`
|
| 4 |
|
| 5 |
## Main Seed Statistics
|
| 6 |
|
|
|
|
| 54 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace | K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 branch trace | 3 | 35.65% +/- 1.22 | +/- 3.02 | 57.16% | 0.406 | +5.91 pp |
|
| 55 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger002 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.02 | 3 | 36.00% +/- 1.25 | +/- 3.12 | 57.42% | 0.407 | +6.26 pp |
|
| 56 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01 | 3 | 36.06% +/- 1.41 | +/- 3.49 | 57.38% | 0.407 | +6.32 pp |
|
| 57 |
+
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_stacknowg | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, no wrong-gripper component on Stack | 0 | missing | missing | missing | missing | missing |
|
| 58 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0005 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.005 | 3 | 36.00% +/- 1.42 | +/- 3.54 | 57.34% | 0.407 | +6.26 pp |
|
| 59 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0015 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.015 | 3 | 36.00% +/- 1.25 | +/- 3.12 | 57.38% | 0.407 | +6.26 pp |
|
| 60 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_scale035 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, scale-gated 0.35 | 3 | 36.00% +/- 1.14 | +/- 2.83 | 57.33% | 0.407 | +6.26 pp |
|
| 61 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_scales035040 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, scale-gated 0.35/0.40 | 3 | 36.00% +/- 1.31 | +/- 3.26 | 57.37% | 0.407 | +6.26 pp |
|
| 62 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger003 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.03 | 3 | 35.94% +/- 1.28 | +/- 3.18 | 57.36% | 0.407 | +6.20 pp |
|
| 63 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01 | 3 | 35.94% +/- 1.13 | +/- 2.81 | 57.40% | 0.424 | +6.20 pp |
|
| 64 |
+
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull | K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only | 3 | 36.00% +/- 1.66 | +/- 4.12 | 57.36% | 0.418 | +6.26 pp |
|
| 65 |
+
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01 | 3 | 36.06% +/- 1.58 | +/- 3.92 | 57.41% | 0.419 | +6.32 pp |
|
| 66 |
| typed_proposal_lattice_types6_prepend_margin000 | Typed proposal lattice head, six families, policy-prepended margin 0.00 | 3 | 31.30% +/- 3.81 | +/- 9.48 | 54.03% | 0.480 | +1.57 pp |
|
| 67 |
| typed_proposal_lattice_types6_prepend_margin005 | Typed proposal lattice head, six families, policy-prepended margin 0.05 | 3 | 32.17% +/- 3.05 | +/- 7.57 | 54.74% | 0.467 | +2.43 pp |
|
| 68 |
| typed_proposal_lattice_types4safe_prepend_margin005 | Typed proposal lattice head, safe four families, policy-prepended margin 0.05 | 3 | 33.45% +/- 3.39 | +/- 8.43 | 55.47% | 0.456 | +3.71 pp |
|
results/paper_table_status.json
CHANGED
|
@@ -1429,6 +1429,25 @@
|
|
| 1429 |
"best_config": null,
|
| 1430 |
"gain_vs_h16_policy": 0.06318840579710144
|
| 1431 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1432 |
{
|
| 1433 |
"key": "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0005",
|
| 1434 |
"label": "K4 compatible residual retrieval, near-miss challenger gate 0.005",
|
|
@@ -1546,21 +1565,40 @@
|
|
| 1546 |
{
|
| 1547 |
"key": "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull",
|
| 1548 |
"label": "K4 compatible residual retrieval, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only",
|
| 1549 |
-
"path": "
|
| 1550 |
"clean_deployment": "yes",
|
| 1551 |
"same_state_proposals": "no",
|
| 1552 |
"expert_proposal": "no",
|
| 1553 |
"story_role": "task-conditional challenger gate using branch trace evidence that wrong-gripper tangents help Pick/Pull but harm other tasks",
|
| 1554 |
"fallback_success": null,
|
| 1555 |
-
"pending_job": "
|
| 1556 |
-
"path_exists":
|
| 1557 |
-
"status": "
|
| 1558 |
-
"success":
|
| 1559 |
-
"std_success":
|
| 1560 |
"completed_seeds": null,
|
| 1561 |
-
"num_completed":
|
| 1562 |
"best_config": null,
|
| 1563 |
-
"gain_vs_h16_policy":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1564 |
},
|
| 1565 |
{
|
| 1566 |
"key": "typed_proposal_lattice_types6_prepend_margin000",
|
|
@@ -2951,23 +2989,23 @@
|
|
| 2951 |
}
|
| 2952 |
],
|
| 2953 |
"best_clean": {
|
| 2954 |
-
"key": "
|
| 2955 |
-
"label": "K4 compatible residual retrieval, near-miss challenger gate 0.01",
|
| 2956 |
-
"path": "
|
| 2957 |
"clean_deployment": "yes",
|
| 2958 |
"same_state_proposals": "no",
|
| 2959 |
"expert_proposal": "no",
|
| 2960 |
-
"story_role": "
|
| 2961 |
"fallback_success": null,
|
| 2962 |
-
"pending_job": "
|
| 2963 |
"path_exists": true,
|
| 2964 |
"status": "complete",
|
| 2965 |
-
"success": 0.
|
| 2966 |
-
"std_success": 0.
|
| 2967 |
"completed_seeds": null,
|
| 2968 |
"num_completed": 3,
|
| 2969 |
"best_config": null,
|
| 2970 |
-
"gain_vs_h16_policy": 0.
|
| 2971 |
},
|
| 2972 |
"best_mechanism_no_expert": {
|
| 2973 |
"key": "no_expert_lattice",
|
|
@@ -2992,7 +3030,7 @@
|
|
| 2992 |
"Use no-expert same-state lattice as the conservative mechanism result, not as deployment-clean inference.",
|
| 2993 |
"Use full lattice only as an upper result because it includes expert proposals.",
|
| 2994 |
"Do not claim external SOTA from this table alone; add current external baselines separately.",
|
| 2995 |
-
"Current best clean deployment row is K4 compatible residual retrieval, near-miss challenger gate 0.01 at 36.06%.",
|
| 2996 |
"Trust-region field optimization should be framed as a negative/diagnostic ablation.",
|
| 2997 |
"Train-state counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.",
|
| 2998 |
"KNN counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.",
|
|
|
|
| 1429 |
"best_config": null,
|
| 1430 |
"gain_vs_h16_policy": 0.06318840579710144
|
| 1431 |
},
|
| 1432 |
+
{
|
| 1433 |
+
"key": "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_stacknowg",
|
| 1434 |
+
"label": "K4 compatible residual retrieval, near-miss challenger gate 0.01, no wrong-gripper component on Stack",
|
| 1435 |
+
"path": "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_stacknowg_summary.json",
|
| 1436 |
+
"clean_deployment": "yes",
|
| 1437 |
+
"same_state_proposals": "no",
|
| 1438 |
+
"expert_proposal": "no",
|
| 1439 |
+
"story_role": "task-local tangent compatibility mask for Stack where wrong-gripper residuals are an anti-affordance",
|
| 1440 |
+
"fallback_success": null,
|
| 1441 |
+
"pending_job": "14976136/14976144",
|
| 1442 |
+
"path_exists": false,
|
| 1443 |
+
"status": "pending",
|
| 1444 |
+
"success": null,
|
| 1445 |
+
"std_success": null,
|
| 1446 |
+
"completed_seeds": null,
|
| 1447 |
+
"num_completed": null,
|
| 1448 |
+
"best_config": null,
|
| 1449 |
+
"gain_vs_h16_policy": null
|
| 1450 |
+
},
|
| 1451 |
{
|
| 1452 |
"key": "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0005",
|
| 1453 |
"label": "K4 compatible residual retrieval, near-miss challenger gate 0.005",
|
|
|
|
| 1565 |
{
|
| 1566 |
"key": "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull",
|
| 1567 |
"label": "K4 compatible residual retrieval, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only",
|
| 1568 |
+
"path": "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgchallenger0p01_pickpull_norm_summary.json",
|
| 1569 |
"clean_deployment": "yes",
|
| 1570 |
"same_state_proposals": "no",
|
| 1571 |
"expert_proposal": "no",
|
| 1572 |
"story_role": "task-conditional challenger gate using branch trace evidence that wrong-gripper tangents help Pick/Pull but harm other tasks",
|
| 1573 |
"fallback_success": null,
|
| 1574 |
+
"pending_job": "14974602/14974608",
|
| 1575 |
+
"path_exists": true,
|
| 1576 |
+
"status": "complete",
|
| 1577 |
+
"success": 0.36,
|
| 1578 |
+
"std_success": 0.01659024698116427,
|
| 1579 |
"completed_seeds": null,
|
| 1580 |
+
"num_completed": 3,
|
| 1581 |
"best_config": null,
|
| 1582 |
+
"gain_vs_h16_policy": 0.06260869565217392
|
| 1583 |
+
},
|
| 1584 |
+
{
|
| 1585 |
+
"key": "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001",
|
| 1586 |
+
"label": "K4 compatible residual retrieval, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01",
|
| 1587 |
+
"path": "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_challenger0p01_norm_summary.json",
|
| 1588 |
+
"clean_deployment": "yes",
|
| 1589 |
+
"same_state_proposals": "no",
|
| 1590 |
+
"expert_proposal": "no",
|
| 1591 |
+
"story_role": "type-specific affordance gate: keep near-miss global while restricting wrong-gripper challenger to acquisition/contact tasks",
|
| 1592 |
+
"fallback_success": null,
|
| 1593 |
+
"pending_job": "14974610/14974611",
|
| 1594 |
+
"path_exists": true,
|
| 1595 |
+
"status": "complete",
|
| 1596 |
+
"success": 0.36057971014492757,
|
| 1597 |
+
"std_success": 0.015780472566743406,
|
| 1598 |
+
"completed_seeds": null,
|
| 1599 |
+
"num_completed": 3,
|
| 1600 |
+
"best_config": null,
|
| 1601 |
+
"gain_vs_h16_policy": 0.0631884057971015
|
| 1602 |
},
|
| 1603 |
{
|
| 1604 |
"key": "typed_proposal_lattice_types6_prepend_margin000",
|
|
|
|
| 2989 |
}
|
| 2990 |
],
|
| 2991 |
"best_clean": {
|
| 2992 |
+
"key": "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001",
|
| 2993 |
+
"label": "K4 compatible residual retrieval, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01",
|
| 2994 |
+
"path": "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_challenger0p01_norm_summary.json",
|
| 2995 |
"clean_deployment": "yes",
|
| 2996 |
"same_state_proposals": "no",
|
| 2997 |
"expert_proposal": "no",
|
| 2998 |
+
"story_role": "type-specific affordance gate: keep near-miss global while restricting wrong-gripper challenger to acquisition/contact tasks",
|
| 2999 |
"fallback_success": null,
|
| 3000 |
+
"pending_job": "14974610/14974611",
|
| 3001 |
"path_exists": true,
|
| 3002 |
"status": "complete",
|
| 3003 |
+
"success": 0.36057971014492757,
|
| 3004 |
+
"std_success": 0.015780472566743406,
|
| 3005 |
"completed_seeds": null,
|
| 3006 |
"num_completed": 3,
|
| 3007 |
"best_config": null,
|
| 3008 |
+
"gain_vs_h16_policy": 0.0631884057971015
|
| 3009 |
},
|
| 3010 |
"best_mechanism_no_expert": {
|
| 3011 |
"key": "no_expert_lattice",
|
|
|
|
| 3030 |
"Use no-expert same-state lattice as the conservative mechanism result, not as deployment-clean inference.",
|
| 3031 |
"Use full lattice only as an upper result because it includes expert proposals.",
|
| 3032 |
"Do not claim external SOTA from this table alone; add current external baselines separately.",
|
| 3033 |
+
"Current best clean deployment row is K4 compatible residual retrieval, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01 at 36.06%.",
|
| 3034 |
"Trust-region field optimization should be framed as a negative/diagnostic ablation.",
|
| 3035 |
"Train-state counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.",
|
| 3036 |
"KNN counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.",
|
results/paper_table_status.md
CHANGED
|
@@ -78,13 +78,15 @@ Baseline h=16 policy: 29.74%
|
|
| 78 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace | K4 compatible residual retrieval, unique candidate-oracle prefix K=8 branch trace | complete | 43.07% | +13.33 pp | diagnostic | no | no | diagnostic branch trace for deployable selector calibration over the local tangent chart |
|
| 79 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger002 | K4 compatible residual retrieval, near-miss challenger gate 0.02 | complete | 36.00% | +6.26 pp | yes | no | no | trace-motivated two-stage selector calibration over the compatible local tangent chart |
|
| 80 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001 | K4 compatible residual retrieval, near-miss challenger gate 0.01 | complete | 36.06% | +6.32 pp | yes | no | no | lower-margin sensitivity for trace-motivated singleton near-miss challenger calibration |
|
|
|
|
| 81 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0005 | K4 compatible residual retrieval, near-miss challenger gate 0.005 | complete | 36.00% | +6.26 pp | yes | no | no | fine lower-margin calibration for the trace-motivated singleton near-miss challenger gate |
|
| 82 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0015 | K4 compatible residual retrieval, near-miss challenger gate 0.015 | complete | 36.00% | +6.26 pp | yes | no | no | fine upper-margin calibration for the trace-motivated singleton near-miss challenger gate |
|
| 83 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_scale035 | K4 compatible residual retrieval, near-miss challenger gate 0.01, scale-gated 0.35 | complete | 36.00% | +6.26 pp | yes | no | no | trace-motivated near-miss challenger with tangent-length reliability gating at the shortest scale |
|
| 84 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_scales035040 | K4 compatible residual retrieval, near-miss challenger gate 0.01, scale-gated 0.35/0.40 | complete | 36.00% | +6.26 pp | yes | no | no | trace-motivated near-miss challenger with tangent-length reliability gating over the two shortest scales |
|
| 85 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger003 | K4 compatible residual retrieval, near-miss challenger gate 0.03 | complete | 35.94% | +6.20 pp | yes | no | no | upper-margin sensitivity for trace-motivated singleton near-miss challenger calibration |
|
| 86 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001 | K4 compatible residual retrieval, near-miss/wrong-gripper challenger gate 0.01 | complete | 35.94% | +6.20 pp | yes | no | no | trace-motivated challenger calibration test for whether wrong-gripper residuals carry conditional selector headroom |
|
| 87 |
-
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull | K4 compatible residual retrieval, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only |
|
|
|
|
| 88 |
| typed_proposal_lattice_types6_prepend_margin000 | Typed proposal lattice head, six families, policy-prepended margin 0.00 | complete | 31.30% | +1.57 pp | yes | no | no | model-generated counterfactual proposal support test; field scores typed proposals rather than retrieved train-state residuals |
|
| 89 |
| typed_proposal_lattice_types6_prepend_margin005 | Typed proposal lattice head, six families, policy-prepended margin 0.05 | complete | 32.17% | +2.43 pp | yes | no | no | model-generated counterfactual proposal support test with a conservative policy-abstention margin |
|
| 90 |
| typed_proposal_lattice_types4safe_prepend_margin005 | Typed proposal lattice head, safe four families, policy-prepended margin 0.05 | complete | 33.45% | +3.71 pp | yes | no | no | anti-goal proposal-family mask test for the model-generated typed lattice |
|
|
@@ -164,7 +166,7 @@ Baseline h=16 policy: 29.74%
|
|
| 164 |
- Use no-expert same-state lattice as the conservative mechanism result, not as deployment-clean inference.
|
| 165 |
- Use full lattice only as an upper result because it includes expert proposals.
|
| 166 |
- Do not claim external SOTA from this table alone; add current external baselines separately.
|
| 167 |
-
- Current best clean deployment row is K4 compatible residual retrieval, near-miss challenger gate 0.01 at 36.06%.
|
| 168 |
- Trust-region field optimization should be framed as a negative/diagnostic ablation.
|
| 169 |
- Train-state counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.
|
| 170 |
- KNN counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.
|
|
|
|
| 78 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace | K4 compatible residual retrieval, unique candidate-oracle prefix K=8 branch trace | complete | 43.07% | +13.33 pp | diagnostic | no | no | diagnostic branch trace for deployable selector calibration over the local tangent chart |
|
| 79 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger002 | K4 compatible residual retrieval, near-miss challenger gate 0.02 | complete | 36.00% | +6.26 pp | yes | no | no | trace-motivated two-stage selector calibration over the compatible local tangent chart |
|
| 80 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001 | K4 compatible residual retrieval, near-miss challenger gate 0.01 | complete | 36.06% | +6.32 pp | yes | no | no | lower-margin sensitivity for trace-motivated singleton near-miss challenger calibration |
|
| 81 |
+
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_stacknowg | K4 compatible residual retrieval, near-miss challenger gate 0.01, no wrong-gripper component on Stack | pending 14976136/14976144 | pending | pending | yes | no | no | task-local tangent compatibility mask for Stack where wrong-gripper residuals are an anti-affordance |
|
| 82 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0005 | K4 compatible residual retrieval, near-miss challenger gate 0.005 | complete | 36.00% | +6.26 pp | yes | no | no | fine lower-margin calibration for the trace-motivated singleton near-miss challenger gate |
|
| 83 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0015 | K4 compatible residual retrieval, near-miss challenger gate 0.015 | complete | 36.00% | +6.26 pp | yes | no | no | fine upper-margin calibration for the trace-motivated singleton near-miss challenger gate |
|
| 84 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_scale035 | K4 compatible residual retrieval, near-miss challenger gate 0.01, scale-gated 0.35 | complete | 36.00% | +6.26 pp | yes | no | no | trace-motivated near-miss challenger with tangent-length reliability gating at the shortest scale |
|
| 85 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_scales035040 | K4 compatible residual retrieval, near-miss challenger gate 0.01, scale-gated 0.35/0.40 | complete | 36.00% | +6.26 pp | yes | no | no | trace-motivated near-miss challenger with tangent-length reliability gating over the two shortest scales |
|
| 86 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger003 | K4 compatible residual retrieval, near-miss challenger gate 0.03 | complete | 35.94% | +6.20 pp | yes | no | no | upper-margin sensitivity for trace-motivated singleton near-miss challenger calibration |
|
| 87 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001 | K4 compatible residual retrieval, near-miss/wrong-gripper challenger gate 0.01 | complete | 35.94% | +6.20 pp | yes | no | no | trace-motivated challenger calibration test for whether wrong-gripper residuals carry conditional selector headroom |
|
| 88 |
+
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull | K4 compatible residual retrieval, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only | complete | 36.00% | +6.26 pp | yes | no | no | task-conditional challenger gate using branch trace evidence that wrong-gripper tangents help Pick/Pull but harm other tasks |
|
| 89 |
+
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001 | K4 compatible residual retrieval, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01 | complete | 36.06% | +6.32 pp | yes | no | no | type-specific affordance gate: keep near-miss global while restricting wrong-gripper challenger to acquisition/contact tasks |
|
| 90 |
| typed_proposal_lattice_types6_prepend_margin000 | Typed proposal lattice head, six families, policy-prepended margin 0.00 | complete | 31.30% | +1.57 pp | yes | no | no | model-generated counterfactual proposal support test; field scores typed proposals rather than retrieved train-state residuals |
|
| 91 |
| typed_proposal_lattice_types6_prepend_margin005 | Typed proposal lattice head, six families, policy-prepended margin 0.05 | complete | 32.17% | +2.43 pp | yes | no | no | model-generated counterfactual proposal support test with a conservative policy-abstention margin |
|
| 92 |
| typed_proposal_lattice_types4safe_prepend_margin005 | Typed proposal lattice head, safe four families, policy-prepended margin 0.05 | complete | 33.45% | +3.71 pp | yes | no | no | anti-goal proposal-family mask test for the model-generated typed lattice |
|
|
|
|
| 166 |
- Use no-expert same-state lattice as the conservative mechanism result, not as deployment-clean inference.
|
| 167 |
- Use full lattice only as an upper result because it includes expert proposals.
|
| 168 |
- Do not claim external SOTA from this table alone; add current external baselines separately.
|
| 169 |
+
- Current best clean deployment row is K4 compatible residual retrieval, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01 at 36.06%.
|
| 170 |
- Trust-region field optimization should be framed as a negative/diagnostic ablation.
|
| 171 |
- Train-state counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.
|
| 172 |
- KNN counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.
|
scripts/build_paper_analysis.py
CHANGED
|
@@ -409,6 +409,14 @@ METHODS = [
|
|
| 409 |
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_summary.json"
|
| 410 |
),
|
| 411 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 412 |
MethodSpec(
|
| 413 |
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0005",
|
| 414 |
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.005",
|
|
@@ -462,7 +470,15 @@ METHODS = [
|
|
| 462 |
label="K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only",
|
| 463 |
summary_path=(
|
| 464 |
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
|
| 465 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 466 |
),
|
| 467 |
),
|
| 468 |
MethodSpec(
|
|
|
|
| 409 |
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_summary.json"
|
| 410 |
),
|
| 411 |
),
|
| 412 |
+
MethodSpec(
|
| 413 |
+
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_stacknowg",
|
| 414 |
+
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, no wrong-gripper component on Stack",
|
| 415 |
+
summary_path=(
|
| 416 |
+
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
|
| 417 |
+
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_stacknowg_summary.json"
|
| 418 |
+
),
|
| 419 |
+
),
|
| 420 |
MethodSpec(
|
| 421 |
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0005",
|
| 422 |
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.005",
|
|
|
|
| 470 |
label="K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only",
|
| 471 |
summary_path=(
|
| 472 |
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
|
| 473 |
+
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgchallenger0p01_pickpull_norm_summary.json"
|
| 474 |
+
),
|
| 475 |
+
),
|
| 476 |
+
MethodSpec(
|
| 477 |
+
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001",
|
| 478 |
+
label="K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01",
|
| 479 |
+
summary_path=(
|
| 480 |
+
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
|
| 481 |
+
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_challenger0p01_norm_summary.json"
|
| 482 |
),
|
| 483 |
),
|
| 484 |
MethodSpec(
|
scripts/build_paper_table_status.py
CHANGED
|
@@ -765,6 +765,16 @@ SPECS = [
|
|
| 765 |
story_role="lower-margin sensitivity for trace-motivated singleton near-miss challenger calibration",
|
| 766 |
pending_job="14954526/14954527",
|
| 767 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 768 |
ResultSpec(
|
| 769 |
key="retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0005",
|
| 770 |
label="K4 compatible residual retrieval, near-miss challenger gate 0.005",
|
|
@@ -828,12 +838,22 @@ SPECS = [
|
|
| 828 |
ResultSpec(
|
| 829 |
key="retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull",
|
| 830 |
label="K4 compatible residual retrieval, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only",
|
| 831 |
-
path="
|
| 832 |
clean_deployment="yes",
|
| 833 |
same_state_proposals="no",
|
| 834 |
expert_proposal="no",
|
| 835 |
story_role="task-conditional challenger gate using branch trace evidence that wrong-gripper tangents help Pick/Pull but harm other tasks",
|
| 836 |
-
pending_job="
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 837 |
),
|
| 838 |
ResultSpec(
|
| 839 |
key="typed_proposal_lattice_types6_prepend_margin000",
|
|
|
|
| 765 |
story_role="lower-margin sensitivity for trace-motivated singleton near-miss challenger calibration",
|
| 766 |
pending_job="14954526/14954527",
|
| 767 |
),
|
| 768 |
+
ResultSpec(
|
| 769 |
+
key="retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_stacknowg",
|
| 770 |
+
label="K4 compatible residual retrieval, near-miss challenger gate 0.01, no wrong-gripper component on Stack",
|
| 771 |
+
path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_stacknowg_summary.json",
|
| 772 |
+
clean_deployment="yes",
|
| 773 |
+
same_state_proposals="no",
|
| 774 |
+
expert_proposal="no",
|
| 775 |
+
story_role="task-local tangent compatibility mask for Stack where wrong-gripper residuals are an anti-affordance",
|
| 776 |
+
pending_job="14976136/14976144",
|
| 777 |
+
),
|
| 778 |
ResultSpec(
|
| 779 |
key="retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0005",
|
| 780 |
label="K4 compatible residual retrieval, near-miss challenger gate 0.005",
|
|
|
|
| 838 |
ResultSpec(
|
| 839 |
key="retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull",
|
| 840 |
label="K4 compatible residual retrieval, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only",
|
| 841 |
+
path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgchallenger0p01_pickpull_norm_summary.json",
|
| 842 |
clean_deployment="yes",
|
| 843 |
same_state_proposals="no",
|
| 844 |
expert_proposal="no",
|
| 845 |
story_role="task-conditional challenger gate using branch trace evidence that wrong-gripper tangents help Pick/Pull but harm other tasks",
|
| 846 |
+
pending_job="14974602/14974608",
|
| 847 |
+
),
|
| 848 |
+
ResultSpec(
|
| 849 |
+
key="retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001",
|
| 850 |
+
label="K4 compatible residual retrieval, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01",
|
| 851 |
+
path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_challenger0p01_norm_summary.json",
|
| 852 |
+
clean_deployment="yes",
|
| 853 |
+
same_state_proposals="no",
|
| 854 |
+
expert_proposal="no",
|
| 855 |
+
story_role="type-specific affordance gate: keep near-miss global while restricting wrong-gripper challenger to acquisition/contact tasks",
|
| 856 |
+
pending_job="14974610/14974611",
|
| 857 |
),
|
| 858 |
ResultSpec(
|
| 859 |
key="typed_proposal_lattice_types6_prepend_margin000",
|
scripts/eval_maniskill_policy_rollout.py
CHANGED
|
@@ -267,12 +267,24 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 267 |
help="Optional comma-separated task IDs where challenger candidates may override "
|
| 268 |
"the primary retrieval-residual selection. Empty enables challengers for all tasks.",
|
| 269 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 270 |
parser.add_argument(
|
| 271 |
"--lattice-exclude-types",
|
| 272 |
default="",
|
| 273 |
help="Comma-separated candidate_type values to mask in selection-mode=lattice, "
|
| 274 |
"for example 'expert'.",
|
| 275 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 276 |
parser.add_argument(
|
| 277 |
"--candidate-type-bonuses",
|
| 278 |
default="",
|
|
@@ -305,6 +317,21 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 305 |
lattice_exclude_types = tuple(
|
| 306 |
item.strip() for item in args.lattice_exclude_types.split(",") if item.strip()
|
| 307 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 308 |
retrieval_residual_challenger_types = tuple(
|
| 309 |
item.strip()
|
| 310 |
for item in args.retrieval_residual_challenger_types.split(",")
|
|
@@ -315,6 +342,21 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 315 |
for item in args.retrieval_residual_challenger_tasks.split(",")
|
| 316 |
if item.strip()
|
| 317 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 318 |
proposal_lattice_types = tuple(
|
| 319 |
item.strip()
|
| 320 |
for item in args.proposal_lattice_types.split(",")
|
|
@@ -402,7 +444,11 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 402 |
retrieval_residual_challenger_scales=retrieval_residual_challenger_scales,
|
| 403 |
retrieval_residual_challenger_margin=args.retrieval_residual_challenger_margin,
|
| 404 |
retrieval_residual_challenger_tasks=retrieval_residual_challenger_tasks,
|
|
|
|
|
|
|
|
|
|
| 405 |
lattice_exclude_types=lattice_exclude_types,
|
|
|
|
| 406 |
candidate_type_bonuses=candidate_type_bonuses,
|
| 407 |
candidate_type_bonus_components=args.candidate_type_bonus_components,
|
| 408 |
candidate_oracle_rollouts=args.candidate_oracle_rollouts,
|
|
|
|
| 267 |
help="Optional comma-separated task IDs where challenger candidates may override "
|
| 268 |
"the primary retrieval-residual selection. Empty enables challengers for all tasks.",
|
| 269 |
)
|
| 270 |
+
parser.add_argument(
|
| 271 |
+
"--retrieval-residual-challenger-type-tasks",
|
| 272 |
+
default="",
|
| 273 |
+
help="Optional semicolon-separated candidate_type=TaskA|TaskB limits for individual "
|
| 274 |
+
"challenger types. Types without an entry keep the global challenger task behavior.",
|
| 275 |
+
)
|
| 276 |
parser.add_argument(
|
| 277 |
"--lattice-exclude-types",
|
| 278 |
default="",
|
| 279 |
help="Comma-separated candidate_type values to mask in selection-mode=lattice, "
|
| 280 |
"for example 'expert'.",
|
| 281 |
)
|
| 282 |
+
parser.add_argument(
|
| 283 |
+
"--lattice-exclude-type-tasks",
|
| 284 |
+
default="",
|
| 285 |
+
help="Optional semicolon-separated candidate_type=TaskA|TaskB exclusions applied "
|
| 286 |
+
"only on listed tasks, e.g. 'wrong_gripper=StackCube-v1'.",
|
| 287 |
+
)
|
| 288 |
parser.add_argument(
|
| 289 |
"--candidate-type-bonuses",
|
| 290 |
default="",
|
|
|
|
| 317 |
lattice_exclude_types = tuple(
|
| 318 |
item.strip() for item in args.lattice_exclude_types.split(",") if item.strip()
|
| 319 |
)
|
| 320 |
+
lattice_exclude_type_tasks: dict[str, tuple[str, ...]] = {}
|
| 321 |
+
try:
|
| 322 |
+
for item in args.lattice_exclude_type_tasks.split(";"):
|
| 323 |
+
item = item.strip()
|
| 324 |
+
if not item:
|
| 325 |
+
continue
|
| 326 |
+
key, value = item.split("=", 1)
|
| 327 |
+
lattice_exclude_type_tasks[key.strip()] = tuple(
|
| 328 |
+
task.strip() for task in value.split("|") if task.strip()
|
| 329 |
+
)
|
| 330 |
+
except ValueError as exc:
|
| 331 |
+
raise SystemExit(
|
| 332 |
+
"--lattice-exclude-type-tasks must contain semicolon-separated "
|
| 333 |
+
"candidate_type=TaskA|TaskB entries"
|
| 334 |
+
) from exc
|
| 335 |
retrieval_residual_challenger_types = tuple(
|
| 336 |
item.strip()
|
| 337 |
for item in args.retrieval_residual_challenger_types.split(",")
|
|
|
|
| 342 |
for item in args.retrieval_residual_challenger_tasks.split(",")
|
| 343 |
if item.strip()
|
| 344 |
)
|
| 345 |
+
retrieval_residual_challenger_type_tasks: dict[str, tuple[str, ...]] = {}
|
| 346 |
+
try:
|
| 347 |
+
for item in args.retrieval_residual_challenger_type_tasks.split(";"):
|
| 348 |
+
item = item.strip()
|
| 349 |
+
if not item:
|
| 350 |
+
continue
|
| 351 |
+
key, value = item.split("=", 1)
|
| 352 |
+
retrieval_residual_challenger_type_tasks[key.strip()] = tuple(
|
| 353 |
+
task.strip() for task in value.split("|") if task.strip()
|
| 354 |
+
)
|
| 355 |
+
except ValueError as exc:
|
| 356 |
+
raise SystemExit(
|
| 357 |
+
"--retrieval-residual-challenger-type-tasks must contain "
|
| 358 |
+
"semicolon-separated candidate_type=TaskA|TaskB entries"
|
| 359 |
+
) from exc
|
| 360 |
proposal_lattice_types = tuple(
|
| 361 |
item.strip()
|
| 362 |
for item in args.proposal_lattice_types.split(",")
|
|
|
|
| 444 |
retrieval_residual_challenger_scales=retrieval_residual_challenger_scales,
|
| 445 |
retrieval_residual_challenger_margin=args.retrieval_residual_challenger_margin,
|
| 446 |
retrieval_residual_challenger_tasks=retrieval_residual_challenger_tasks,
|
| 447 |
+
retrieval_residual_challenger_type_tasks=(
|
| 448 |
+
retrieval_residual_challenger_type_tasks
|
| 449 |
+
),
|
| 450 |
lattice_exclude_types=lattice_exclude_types,
|
| 451 |
+
lattice_exclude_type_tasks=lattice_exclude_type_tasks,
|
| 452 |
candidate_type_bonuses=candidate_type_bonuses,
|
| 453 |
candidate_type_bonus_components=args.candidate_type_bonus_components,
|
| 454 |
candidate_oracle_rollouts=args.candidate_oracle_rollouts,
|
scripts/slurm/eval_maniskill_policy_rollout.sbatch
CHANGED
|
@@ -87,10 +87,12 @@ RETRIEVAL_RESIDUAL_CHALLENGER_TASKS="${RETRIEVAL_RESIDUAL_CHALLENGER_TASKS:-}"
|
|
| 87 |
if [[ -n "${RETRIEVAL_RESIDUAL_CHALLENGER_TASKS_COLON:-}" ]]; then
|
| 88 |
RETRIEVAL_RESIDUAL_CHALLENGER_TASKS="${RETRIEVAL_RESIDUAL_CHALLENGER_TASKS_COLON//:/,}"
|
| 89 |
fi
|
|
|
|
| 90 |
LATTICE_EXCLUDE_TYPES="${LATTICE_EXCLUDE_TYPES:-}"
|
| 91 |
if [[ -n "${LATTICE_EXCLUDE_TYPES_COLON:-}" ]]; then
|
| 92 |
LATTICE_EXCLUDE_TYPES="${LATTICE_EXCLUDE_TYPES_COLON//:/,}"
|
| 93 |
fi
|
|
|
|
| 94 |
CANDIDATE_TYPE_BONUSES="${CANDIDATE_TYPE_BONUSES:-}"
|
| 95 |
if [[ -n "${CANDIDATE_TYPE_BONUSES_COLON:-}" ]]; then
|
| 96 |
CANDIDATE_TYPE_BONUSES="${CANDIDATE_TYPE_BONUSES_COLON//:/,}"
|
|
@@ -170,7 +172,9 @@ apptainer exec --nv \
|
|
| 170 |
--retrieval-residual-challenger-scales "$RETRIEVAL_RESIDUAL_CHALLENGER_SCALES" \
|
| 171 |
--retrieval-residual-challenger-margin "$RETRIEVAL_RESIDUAL_CHALLENGER_MARGIN" \
|
| 172 |
--retrieval-residual-challenger-tasks "$RETRIEVAL_RESIDUAL_CHALLENGER_TASKS" \
|
|
|
|
| 173 |
--lattice-exclude-types "$LATTICE_EXCLUDE_TYPES" \
|
|
|
|
| 174 |
--candidate-type-bonuses "$CANDIDATE_TYPE_BONUSES" \
|
| 175 |
--candidate-oracle-rollouts "$CANDIDATE_ORACLE_ROLLOUTS" \
|
| 176 |
--candidate-oracle-unique-tolerance "$CANDIDATE_ORACLE_UNIQUE_TOLERANCE" \
|
|
|
|
| 87 |
if [[ -n "${RETRIEVAL_RESIDUAL_CHALLENGER_TASKS_COLON:-}" ]]; then
|
| 88 |
RETRIEVAL_RESIDUAL_CHALLENGER_TASKS="${RETRIEVAL_RESIDUAL_CHALLENGER_TASKS_COLON//:/,}"
|
| 89 |
fi
|
| 90 |
+
RETRIEVAL_RESIDUAL_CHALLENGER_TYPE_TASKS="${RETRIEVAL_RESIDUAL_CHALLENGER_TYPE_TASKS:-}"
|
| 91 |
LATTICE_EXCLUDE_TYPES="${LATTICE_EXCLUDE_TYPES:-}"
|
| 92 |
if [[ -n "${LATTICE_EXCLUDE_TYPES_COLON:-}" ]]; then
|
| 93 |
LATTICE_EXCLUDE_TYPES="${LATTICE_EXCLUDE_TYPES_COLON//:/,}"
|
| 94 |
fi
|
| 95 |
+
LATTICE_EXCLUDE_TYPE_TASKS="${LATTICE_EXCLUDE_TYPE_TASKS:-}"
|
| 96 |
CANDIDATE_TYPE_BONUSES="${CANDIDATE_TYPE_BONUSES:-}"
|
| 97 |
if [[ -n "${CANDIDATE_TYPE_BONUSES_COLON:-}" ]]; then
|
| 98 |
CANDIDATE_TYPE_BONUSES="${CANDIDATE_TYPE_BONUSES_COLON//:/,}"
|
|
|
|
| 172 |
--retrieval-residual-challenger-scales "$RETRIEVAL_RESIDUAL_CHALLENGER_SCALES" \
|
| 173 |
--retrieval-residual-challenger-margin "$RETRIEVAL_RESIDUAL_CHALLENGER_MARGIN" \
|
| 174 |
--retrieval-residual-challenger-tasks "$RETRIEVAL_RESIDUAL_CHALLENGER_TASKS" \
|
| 175 |
+
--retrieval-residual-challenger-type-tasks "$RETRIEVAL_RESIDUAL_CHALLENGER_TYPE_TASKS" \
|
| 176 |
--lattice-exclude-types "$LATTICE_EXCLUDE_TYPES" \
|
| 177 |
+
--lattice-exclude-type-tasks "$LATTICE_EXCLUDE_TYPE_TASKS" \
|
| 178 |
--candidate-type-bonuses "$CANDIDATE_TYPE_BONUSES" \
|
| 179 |
--candidate-oracle-rollouts "$CANDIDATE_ORACLE_ROLLOUTS" \
|
| 180 |
--candidate-oracle-unique-tolerance "$CANDIDATE_ORACLE_UNIQUE_TOLERANCE" \
|
scripts/slurm/summarize_h16_policy_ckpt.sbatch
CHANGED
|
@@ -115,7 +115,11 @@ for result_path in sorted(base_dir.glob(f"seed_*/{out_name}")):
|
|
| 115 |
"retrieval_residual_challenger_tasks": data.get(
|
| 116 |
"retrieval_residual_challenger_tasks", []
|
| 117 |
),
|
|
|
|
|
|
|
|
|
|
| 118 |
"lattice_exclude_types": data.get("lattice_exclude_types", []),
|
|
|
|
| 119 |
"candidate_type_bonuses": data.get("candidate_type_bonuses", {}),
|
| 120 |
"candidate_type_bonus_components": data.get(
|
| 121 |
"candidate_type_bonus_components", False
|
|
@@ -401,6 +405,26 @@ for row in rows:
|
|
| 401 |
if challenger_tasks
|
| 402 |
else ""
|
| 403 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 404 |
lines.append(
|
| 405 |
"| {seed} | {mode} | {k} | {policy_cand} | {retrieval} | {metric} | {anchor} | {direction} | {reduce}{challenger} | {min_success:.2f} | {type_success_bonus:.3f} | {consensus_penalty:.3f} | {min_source_progress:.2f} | {source_progress_bonus:.3f} | {source_score_bonus:.3f} | {scale:.2f} | {scale_grid} | {margin:.3f} | {sigma:.2f} | {steps} | {trust:.2f} | "
|
| 406 |
"{success:.2%} | {progress:.2%} | {oracle:.2%} | {candidate_oracle} | {oracle_gain} | {mse:.3f} |".format(
|
|
@@ -413,7 +437,8 @@ for row in rows:
|
|
| 413 |
anchor=row.get("retrieval_residual_anchor") or "none",
|
| 414 |
direction=row.get("retrieval_residual_direction")
|
| 415 |
or "candidate_minus_anchor",
|
| 416 |
-
reduce=row.get("retrieval_residual_reduce") or "none"
|
|
|
|
| 417 |
challenger=(
|
| 418 |
" + challenger "
|
| 419 |
+ ",".join(row.get("retrieval_residual_challenger_types") or [])
|
|
@@ -428,6 +453,7 @@ for row in rows:
|
|
| 428 |
else ""
|
| 429 |
)
|
| 430 |
+ challenger_task_suffix
|
|
|
|
| 431 |
+ f"@{row.get('retrieval_residual_challenger_margin') or 0.0:.2f}"
|
| 432 |
if row.get("retrieval_residual_challenger_types")
|
| 433 |
else ""
|
|
|
|
| 115 |
"retrieval_residual_challenger_tasks": data.get(
|
| 116 |
"retrieval_residual_challenger_tasks", []
|
| 117 |
),
|
| 118 |
+
"retrieval_residual_challenger_type_tasks": data.get(
|
| 119 |
+
"retrieval_residual_challenger_type_tasks", {}
|
| 120 |
+
),
|
| 121 |
"lattice_exclude_types": data.get("lattice_exclude_types", []),
|
| 122 |
+
"lattice_exclude_type_tasks": data.get("lattice_exclude_type_tasks", {}),
|
| 123 |
"candidate_type_bonuses": data.get("candidate_type_bonuses", {}),
|
| 124 |
"candidate_type_bonus_components": data.get(
|
| 125 |
"candidate_type_bonus_components", False
|
|
|
|
| 405 |
if challenger_tasks
|
| 406 |
else ""
|
| 407 |
)
|
| 408 |
+
challenger_type_tasks = row.get("retrieval_residual_challenger_type_tasks") or {}
|
| 409 |
+
challenger_type_task_suffix = (
|
| 410 |
+
" with "
|
| 411 |
+
+ ";".join(
|
| 412 |
+
f"{candidate_type}=" + ",".join(tasks)
|
| 413 |
+
for candidate_type, tasks in sorted(challenger_type_tasks.items())
|
| 414 |
+
)
|
| 415 |
+
if challenger_type_tasks
|
| 416 |
+
else ""
|
| 417 |
+
)
|
| 418 |
+
exclude_type_tasks = row.get("lattice_exclude_type_tasks") or {}
|
| 419 |
+
exclude_type_task_suffix = (
|
| 420 |
+
" mask "
|
| 421 |
+
+ ";".join(
|
| 422 |
+
f"{candidate_type}=" + ",".join(tasks)
|
| 423 |
+
for candidate_type, tasks in sorted(exclude_type_tasks.items())
|
| 424 |
+
)
|
| 425 |
+
if exclude_type_tasks
|
| 426 |
+
else ""
|
| 427 |
+
)
|
| 428 |
lines.append(
|
| 429 |
"| {seed} | {mode} | {k} | {policy_cand} | {retrieval} | {metric} | {anchor} | {direction} | {reduce}{challenger} | {min_success:.2f} | {type_success_bonus:.3f} | {consensus_penalty:.3f} | {min_source_progress:.2f} | {source_progress_bonus:.3f} | {source_score_bonus:.3f} | {scale:.2f} | {scale_grid} | {margin:.3f} | {sigma:.2f} | {steps} | {trust:.2f} | "
|
| 430 |
"{success:.2%} | {progress:.2%} | {oracle:.2%} | {candidate_oracle} | {oracle_gain} | {mse:.3f} |".format(
|
|
|
|
| 437 |
anchor=row.get("retrieval_residual_anchor") or "none",
|
| 438 |
direction=row.get("retrieval_residual_direction")
|
| 439 |
or "candidate_minus_anchor",
|
| 440 |
+
reduce=(row.get("retrieval_residual_reduce") or "none")
|
| 441 |
+
+ exclude_type_task_suffix,
|
| 442 |
challenger=(
|
| 443 |
" + challenger "
|
| 444 |
+ ",".join(row.get("retrieval_residual_challenger_types") or [])
|
|
|
|
| 453 |
else ""
|
| 454 |
)
|
| 455 |
+ challenger_task_suffix
|
| 456 |
+
+ challenger_type_task_suffix
|
| 457 |
+ f"@{row.get('retrieval_residual_challenger_margin') or 0.0:.2f}"
|
| 458 |
if row.get("retrieval_residual_challenger_types")
|
| 459 |
else ""
|