Auto-sync: 2026-06-30 23:06:36
Browse files- results/paper_analysis.json +198 -4
- results/paper_analysis.md +5 -3
- results/paper_table_status.json +43 -5
- results/paper_table_status.md +4 -2
- scripts/build_paper_analysis.py +16 -0
- scripts/build_paper_table_status.py +20 -0
results/paper_analysis.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"best_clean_key": "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001",
|
| 3 |
-
"generated_utc": "2026-07-
|
| 4 |
"mechanism_gap": {
|
| 5 |
"best_clean_vs_direct_same_ckpt": 0.0776811594202898,
|
| 6 |
"best_clean_vs_h16": 0.06318840579710144,
|
|
@@ -2208,6 +2208,11 @@
|
|
| 2208 |
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p03_summary.json",
|
| 2209 |
"std_success": 0.012819330079707819
|
| 2210 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2211 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001": {
|
| 2212 |
"ci95_success": 0.03920403094295293,
|
| 2213 |
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01",
|
|
@@ -2306,14 +2311,203 @@
|
|
| 2306 |
"std_success": 0.015780472566743406
|
| 2307 |
},
|
| 2308 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin003_challenger001": {
|
|
|
|
| 2309 |
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper Pick/Pull margin 0.03",
|
| 2310 |
-
"
|
| 2311 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2312 |
},
|
| 2313 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin005_challenger001": {
|
|
|
|
| 2314 |
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper Pick/Pull margin 0.05",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2315 |
"missing": true,
|
| 2316 |
-
"source": "results/
|
| 2317 |
},
|
| 2318 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001": {
|
| 2319 |
"ci95_success": 0.028111505659259583,
|
|
|
|
| 1 |
{
|
| 2 |
"best_clean_key": "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001",
|
| 3 |
+
"generated_utc": "2026-07-01T03:07:08+00:00",
|
| 4 |
"mechanism_gap": {
|
| 5 |
"best_clean_vs_direct_same_ckpt": 0.0776811594202898,
|
| 6 |
"best_clean_vs_h16": 0.06318840579710144,
|
|
|
|
| 2208 |
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p03_summary.json",
|
| 2209 |
"std_success": 0.012819330079707819
|
| 2210 |
},
|
| 2211 |
+
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_noopwgcontact_challenger001": {
|
| 2212 |
+
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss global + no-op/wrong-gripper contact challenger",
|
| 2213 |
+
"missing": true,
|
| 2214 |
+
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_noopwgcontact_challenger0p01_summary.json"
|
| 2215 |
+
},
|
| 2216 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001": {
|
| 2217 |
"ci95_success": 0.03920403094295293,
|
| 2218 |
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01",
|
|
|
|
| 2311 |
"std_success": 0.015780472566743406
|
| 2312 |
},
|
| 2313 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin003_challenger001": {
|
| 2314 |
+
"ci95_success": 0.039598844840038006,
|
| 2315 |
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper Pick/Pull margin 0.03",
|
| 2316 |
+
"mean_action_mse_to_best": 0.41668552688795374,
|
| 2317 |
+
"mean_progress": 0.5738671411409219,
|
| 2318 |
+
"mean_success": 0.36000000000000004,
|
| 2319 |
+
"num_completed": 3,
|
| 2320 |
+
"per_task_success": {
|
| 2321 |
+
"LiftPegUpright-v1": {
|
| 2322 |
+
"mean_num_groups": 102.0,
|
| 2323 |
+
"mean_success": 0.26340865087329823,
|
| 2324 |
+
"std_success": 0.06057881385012102
|
| 2325 |
+
},
|
| 2326 |
+
"PickCube-v1": {
|
| 2327 |
+
"mean_num_groups": 196.66666666666666,
|
| 2328 |
+
"mean_success": 0.3289542583020844,
|
| 2329 |
+
"std_success": 0.04030135158178743
|
| 2330 |
+
},
|
| 2331 |
+
"PullCube-v1": {
|
| 2332 |
+
"mean_num_groups": 81.0,
|
| 2333 |
+
"mean_success": 0.228271436166173,
|
| 2334 |
+
"std_success": 0.03525140677829959
|
| 2335 |
+
},
|
| 2336 |
+
"PushCube-v1": {
|
| 2337 |
+
"mean_num_groups": 102.0,
|
| 2338 |
+
"mean_success": 0.7604066042026859,
|
| 2339 |
+
"std_success": 0.08151850231627764
|
| 2340 |
+
},
|
| 2341 |
+
"StackCube-v1": {
|
| 2342 |
+
"mean_num_groups": 93.33333333333333,
|
| 2343 |
+
"mean_success": 0.19707329707329704,
|
| 2344 |
+
"std_success": 0.0831467196651622
|
| 2345 |
+
}
|
| 2346 |
+
},
|
| 2347 |
+
"seed_action_mse_to_best": {
|
| 2348 |
+
"0": 0.39585299389679796,
|
| 2349 |
+
"1": 0.40845387818858675,
|
| 2350 |
+
"2": 0.44574970857847646
|
| 2351 |
+
},
|
| 2352 |
+
"seed_progress": {
|
| 2353 |
+
"0": 0.5592340455844027,
|
| 2354 |
+
"1": 0.5719369965759309,
|
| 2355 |
+
"2": 0.590430381262432
|
| 2356 |
+
},
|
| 2357 |
+
"seed_success": {
|
| 2358 |
+
"0": 0.3565217391304348,
|
| 2359 |
+
"1": 0.34608695652173915,
|
| 2360 |
+
"2": 0.3773913043478261
|
| 2361 |
+
},
|
| 2362 |
+
"selected_candidate_type_counts": {
|
| 2363 |
+
"retrieval_residual_policy_residual": 1476,
|
| 2364 |
+
"retrieval_residual_residual_near_miss": 91,
|
| 2365 |
+
"retrieval_residual_residual_near_miss+residual_wrong_gripper": 5,
|
| 2366 |
+
"retrieval_residual_residual_no_op": 38,
|
| 2367 |
+
"retrieval_residual_residual_no_op+residual_wrong_gripper": 27,
|
| 2368 |
+
"retrieval_residual_residual_wrong_gripper": 88
|
| 2369 |
+
},
|
| 2370 |
+
"selected_residual_scale_counts": {
|
| 2371 |
+
"0.35": 1530,
|
| 2372 |
+
"0.4": 29,
|
| 2373 |
+
"0.45": 166
|
| 2374 |
+
},
|
| 2375 |
+
"selected_type_outcomes": {
|
| 2376 |
+
"retrieval_residual_policy_residual": {
|
| 2377 |
+
"count": 1476.0,
|
| 2378 |
+
"mean_progress": 0.5566651973590068,
|
| 2379 |
+
"success_rate": 0.3346883468834688
|
| 2380 |
+
},
|
| 2381 |
+
"retrieval_residual_residual_near_miss": {
|
| 2382 |
+
"count": 91.0,
|
| 2383 |
+
"mean_progress": 0.5661385127926594,
|
| 2384 |
+
"success_rate": 0.38461538461538464
|
| 2385 |
+
},
|
| 2386 |
+
"retrieval_residual_residual_near_miss+residual_wrong_gripper": {
|
| 2387 |
+
"count": 5.0,
|
| 2388 |
+
"mean_progress": 0.7557853102684021,
|
| 2389 |
+
"success_rate": 0.6
|
| 2390 |
+
},
|
| 2391 |
+
"retrieval_residual_residual_no_op": {
|
| 2392 |
+
"count": 38.0,
|
| 2393 |
+
"mean_progress": 0.7757954887653652,
|
| 2394 |
+
"success_rate": 0.5263157894736842
|
| 2395 |
+
},
|
| 2396 |
+
"retrieval_residual_residual_no_op+residual_wrong_gripper": {
|
| 2397 |
+
"count": 27.0,
|
| 2398 |
+
"mean_progress": 0.6916872307106301,
|
| 2399 |
+
"success_rate": 0.48148148148148145
|
| 2400 |
+
},
|
| 2401 |
+
"retrieval_residual_residual_wrong_gripper": {
|
| 2402 |
+
"count": 88.0,
|
| 2403 |
+
"mean_progress": 0.7367008198687638,
|
| 2404 |
+
"success_rate": 0.6363636363636364
|
| 2405 |
+
}
|
| 2406 |
+
},
|
| 2407 |
+
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_wgmargin0p03_challenger0p01_summary.json",
|
| 2408 |
+
"std_success": 0.01593939372158552
|
| 2409 |
},
|
| 2410 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin005_challenger001": {
|
| 2411 |
+
"ci95_success": 0.03920403094295293,
|
| 2412 |
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper Pick/Pull margin 0.05",
|
| 2413 |
+
"mean_action_mse_to_best": 0.41403828773242624,
|
| 2414 |
+
"mean_progress": 0.5730632333303599,
|
| 2415 |
+
"mean_success": 0.3588405797101449,
|
| 2416 |
+
"num_completed": 3,
|
| 2417 |
+
"per_task_success": {
|
| 2418 |
+
"LiftPegUpright-v1": {
|
| 2419 |
+
"mean_num_groups": 102.0,
|
| 2420 |
+
"mean_success": 0.26340865087329823,
|
| 2421 |
+
"std_success": 0.06057881385012102
|
| 2422 |
+
},
|
| 2423 |
+
"PickCube-v1": {
|
| 2424 |
+
"mean_num_groups": 196.66666666666666,
|
| 2425 |
+
"mean_success": 0.3272707566185827,
|
| 2426 |
+
"std_success": 0.03916710518683381
|
| 2427 |
+
},
|
| 2428 |
+
"PullCube-v1": {
|
| 2429 |
+
"mean_num_groups": 81.0,
|
| 2430 |
+
"mean_success": 0.22388547125389233,
|
| 2431 |
+
"std_success": 0.04131573991107309
|
| 2432 |
+
},
|
| 2433 |
+
"PushCube-v1": {
|
| 2434 |
+
"mean_num_groups": 102.0,
|
| 2435 |
+
"mean_success": 0.7604066042026859,
|
| 2436 |
+
"std_success": 0.08151850231627764
|
| 2437 |
+
},
|
| 2438 |
+
"StackCube-v1": {
|
| 2439 |
+
"mean_num_groups": 93.33333333333333,
|
| 2440 |
+
"mean_success": 0.19707329707329704,
|
| 2441 |
+
"std_success": 0.0831467196651622
|
| 2442 |
+
}
|
| 2443 |
+
},
|
| 2444 |
+
"seed_action_mse_to_best": {
|
| 2445 |
+
"0": 0.3946219221810284,
|
| 2446 |
+
"1": 0.4068148876516067,
|
| 2447 |
+
"2": 0.4406780533646436
|
| 2448 |
+
},
|
| 2449 |
+
"seed_progress": {
|
| 2450 |
+
"0": 0.5592912306045384,
|
| 2451 |
+
"1": 0.5702725335210562,
|
| 2452 |
+
"2": 0.5896259358654852
|
| 2453 |
+
},
|
| 2454 |
+
"seed_success": {
|
| 2455 |
+
"0": 0.3565217391304348,
|
| 2456 |
+
"1": 0.3443478260869565,
|
| 2457 |
+
"2": 0.37565217391304345
|
| 2458 |
+
},
|
| 2459 |
+
"selected_candidate_type_counts": {
|
| 2460 |
+
"retrieval_residual_policy_residual": 1503,
|
| 2461 |
+
"retrieval_residual_residual_near_miss": 91,
|
| 2462 |
+
"retrieval_residual_residual_near_miss+residual_wrong_gripper": 5,
|
| 2463 |
+
"retrieval_residual_residual_no_op": 38,
|
| 2464 |
+
"retrieval_residual_residual_no_op+residual_wrong_gripper": 27,
|
| 2465 |
+
"retrieval_residual_residual_wrong_gripper": 61
|
| 2466 |
+
},
|
| 2467 |
+
"selected_residual_scale_counts": {
|
| 2468 |
+
"0.35": 1550,
|
| 2469 |
+
"0.4": 26,
|
| 2470 |
+
"0.45": 149
|
| 2471 |
+
},
|
| 2472 |
+
"selected_type_outcomes": {
|
| 2473 |
+
"retrieval_residual_policy_residual": {
|
| 2474 |
+
"count": 1503.0,
|
| 2475 |
+
"mean_progress": 0.5571624199836251,
|
| 2476 |
+
"success_rate": 0.3366600133067199
|
| 2477 |
+
},
|
| 2478 |
+
"retrieval_residual_residual_near_miss": {
|
| 2479 |
+
"count": 91.0,
|
| 2480 |
+
"mean_progress": 0.5661385127926594,
|
| 2481 |
+
"success_rate": 0.38461538461538464
|
| 2482 |
+
},
|
| 2483 |
+
"retrieval_residual_residual_near_miss+residual_wrong_gripper": {
|
| 2484 |
+
"count": 5.0,
|
| 2485 |
+
"mean_progress": 0.7557853102684021,
|
| 2486 |
+
"success_rate": 0.6
|
| 2487 |
+
},
|
| 2488 |
+
"retrieval_residual_residual_no_op": {
|
| 2489 |
+
"count": 38.0,
|
| 2490 |
+
"mean_progress": 0.7757954887653652,
|
| 2491 |
+
"success_rate": 0.5263157894736842
|
| 2492 |
+
},
|
| 2493 |
+
"retrieval_residual_residual_no_op+residual_wrong_gripper": {
|
| 2494 |
+
"count": 27.0,
|
| 2495 |
+
"mean_progress": 0.6916872307106301,
|
| 2496 |
+
"success_rate": 0.48148148148148145
|
| 2497 |
+
},
|
| 2498 |
+
"retrieval_residual_residual_wrong_gripper": {
|
| 2499 |
+
"count": 61.0,
|
| 2500 |
+
"mean_progress": 0.7814040203563503,
|
| 2501 |
+
"success_rate": 0.6885245901639344
|
| 2502 |
+
}
|
| 2503 |
+
},
|
| 2504 |
+
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_wgmargin0p05_challenger0p01_summary.json",
|
| 2505 |
+
"std_success": 0.015780472566743406
|
| 2506 |
+
},
|
| 2507 |
+
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpullstack_challenger001": {
|
| 2508 |
+
"label": "K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull/Stack",
|
| 2509 |
"missing": true,
|
| 2510 |
+
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpullstack_challenger0p01_summary.json"
|
| 2511 |
},
|
| 2512 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001": {
|
| 2513 |
"ci95_success": 0.028111505659259583,
|
results/paper_analysis.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
# Paper Analysis
|
| 2 |
|
| 3 |
-
Generated: `2026-07-
|
| 4 |
|
| 5 |
## Main Seed Statistics
|
| 6 |
|
|
@@ -65,8 +65,10 @@ Generated: `2026-07-01T01:39:11+00:00`
|
|
| 65 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgmargin005_challenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger 0.01, wrong-gripper margin 0.05 | 3 | 35.59% +/- 1.31 | +/- 3.24 | 57.14% | 0.417 | +5.86 pp |
|
| 66 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull | K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only | 3 | 36.00% +/- 1.66 | +/- 4.12 | 57.36% | 0.418 | +6.26 pp |
|
| 67 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01 | 3 | 36.06% +/- 1.58 | +/- 3.92 | 57.41% | 0.419 | +6.32 pp |
|
| 68 |
-
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin003_challenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper Pick/Pull margin 0.03 |
|
| 69 |
-
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin005_challenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper Pick/Pull margin 0.05 |
|
|
|
|
|
|
|
| 70 |
| typed_proposal_lattice_types6_prepend_margin000 | Typed proposal lattice head, six families, policy-prepended margin 0.00 | 3 | 31.30% +/- 3.81 | +/- 9.48 | 54.03% | 0.480 | +1.57 pp |
|
| 71 |
| typed_proposal_lattice_types6_prepend_margin005 | Typed proposal lattice head, six families, policy-prepended margin 0.05 | 3 | 32.17% +/- 3.05 | +/- 7.57 | 54.74% | 0.467 | +2.43 pp |
|
| 72 |
| typed_proposal_lattice_types4safe_prepend_margin005 | Typed proposal lattice head, safe four families, policy-prepended margin 0.05 | 3 | 33.45% +/- 3.39 | +/- 8.43 | 55.47% | 0.456 | +3.71 pp |
|
|
|
|
| 1 |
# Paper Analysis
|
| 2 |
|
| 3 |
+
Generated: `2026-07-01T03:07:08+00:00`
|
| 4 |
|
| 5 |
## Main Seed Statistics
|
| 6 |
|
|
|
|
| 65 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgmargin005_challenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss challenger 0.01, wrong-gripper margin 0.05 | 3 | 35.59% +/- 1.31 | +/- 3.24 | 57.14% | 0.417 | +5.86 pp |
|
| 66 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull | K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only | 3 | 36.00% +/- 1.66 | +/- 4.12 | 57.36% | 0.418 | +6.26 pp |
|
| 67 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01 | 3 | 36.06% +/- 1.58 | +/- 3.92 | 57.41% | 0.419 | +6.32 pp |
|
| 68 |
+
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin003_challenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper Pick/Pull margin 0.03 | 3 | 36.00% +/- 1.59 | +/- 3.96 | 57.39% | 0.417 | +6.26 pp |
|
| 69 |
+
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin005_challenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper Pick/Pull margin 0.05 | 3 | 35.88% +/- 1.58 | +/- 3.92 | 57.31% | 0.414 | +6.14 pp |
|
| 70 |
+
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpullstack_challenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull/Stack | 0 | missing | missing | missing | missing | missing |
|
| 71 |
+
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_noopwgcontact_challenger001 | K4 compatible tangents, no-op bonus 0.03, near-miss global + no-op/wrong-gripper contact challenger | 0 | missing | missing | missing | missing | missing |
|
| 72 |
| typed_proposal_lattice_types6_prepend_margin000 | Typed proposal lattice head, six families, policy-prepended margin 0.00 | 3 | 31.30% +/- 3.81 | +/- 9.48 | 54.03% | 0.480 | +1.57 pp |
|
| 73 |
| typed_proposal_lattice_types6_prepend_margin005 | Typed proposal lattice head, six families, policy-prepended margin 0.05 | 3 | 32.17% +/- 3.05 | +/- 7.57 | 54.74% | 0.467 | +2.43 pp |
|
| 74 |
| typed_proposal_lattice_types4safe_prepend_margin005 | Typed proposal lattice head, safe four families, policy-prepended margin 0.05 | 3 | 33.45% +/- 3.39 | +/- 8.43 | 55.47% | 0.456 | +3.71 pp |
|
results/paper_table_status.json
CHANGED
|
@@ -1648,6 +1648,44 @@
|
|
| 1648 |
"story_role": "type-specific affordance margin: keep near-miss globally sensitive while requiring stronger wrong-gripper evidence only on Pick/Pull",
|
| 1649 |
"fallback_success": null,
|
| 1650 |
"pending_job": "14991117/14991125",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1651 |
"path_exists": false,
|
| 1652 |
"status": "pending",
|
| 1653 |
"success": null,
|
|
@@ -1658,15 +1696,15 @@
|
|
| 1658 |
"gain_vs_h16_policy": null
|
| 1659 |
},
|
| 1660 |
{
|
| 1661 |
-
"key": "
|
| 1662 |
-
"label": "K4 compatible residual retrieval, near-miss global + wrong-gripper
|
| 1663 |
-
"path": "
|
| 1664 |
"clean_deployment": "yes",
|
| 1665 |
"same_state_proposals": "no",
|
| 1666 |
"expert_proposal": "no",
|
| 1667 |
-
"story_role": "
|
| 1668 |
"fallback_success": null,
|
| 1669 |
-
"pending_job": "
|
| 1670 |
"path_exists": false,
|
| 1671 |
"status": "pending",
|
| 1672 |
"success": null,
|
|
|
|
| 1648 |
"story_role": "type-specific affordance margin: keep near-miss globally sensitive while requiring stronger wrong-gripper evidence only on Pick/Pull",
|
| 1649 |
"fallback_success": null,
|
| 1650 |
"pending_job": "14991117/14991125",
|
| 1651 |
+
"path_exists": true,
|
| 1652 |
+
"status": "complete",
|
| 1653 |
+
"success": 0.36,
|
| 1654 |
+
"std_success": 0.01593939372158552,
|
| 1655 |
+
"completed_seeds": null,
|
| 1656 |
+
"num_completed": 3,
|
| 1657 |
+
"best_config": null,
|
| 1658 |
+
"gain_vs_h16_policy": 0.06260869565217392
|
| 1659 |
+
},
|
| 1660 |
+
{
|
| 1661 |
+
"key": "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin005_challenger001",
|
| 1662 |
+
"label": "K4 compatible residual retrieval, near-miss global + wrong-gripper Pick/Pull margin 0.05",
|
| 1663 |
+
"path": "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_wgmargin0p05_challenger0p01_summary.json",
|
| 1664 |
+
"clean_deployment": "yes",
|
| 1665 |
+
"same_state_proposals": "no",
|
| 1666 |
+
"expert_proposal": "no",
|
| 1667 |
+
"story_role": "upper setting for type-specific wrong-gripper evidence on Pick/Pull while preserving global near-miss calibration",
|
| 1668 |
+
"fallback_success": null,
|
| 1669 |
+
"pending_job": "14991123/14991126",
|
| 1670 |
+
"path_exists": true,
|
| 1671 |
+
"status": "complete",
|
| 1672 |
+
"success": 0.35884057971014494,
|
| 1673 |
+
"std_success": 0.015780472566743406,
|
| 1674 |
+
"completed_seeds": null,
|
| 1675 |
+
"num_completed": 3,
|
| 1676 |
+
"best_config": null,
|
| 1677 |
+
"gain_vs_h16_policy": 0.06144927536231887
|
| 1678 |
+
},
|
| 1679 |
+
{
|
| 1680 |
+
"key": "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpullstack_challenger001",
|
| 1681 |
+
"label": "K4 compatible residual retrieval, near-miss global + wrong-gripper challenger on Pick/Pull/Stack",
|
| 1682 |
+
"path": "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpullstack_challenger0p01_summary.json",
|
| 1683 |
+
"clean_deployment": "yes",
|
| 1684 |
+
"same_state_proposals": "no",
|
| 1685 |
+
"expert_proposal": "no",
|
| 1686 |
+
"story_role": "oracle-trace contact-task calibration: wrong-gripper residuals are only challenger-eligible on Pick/Pull/Stack while near-miss stays global",
|
| 1687 |
+
"fallback_success": null,
|
| 1688 |
+
"pending_job": "14995028/14995047",
|
| 1689 |
"path_exists": false,
|
| 1690 |
"status": "pending",
|
| 1691 |
"success": null,
|
|
|
|
| 1696 |
"gain_vs_h16_policy": null
|
| 1697 |
},
|
| 1698 |
{
|
| 1699 |
+
"key": "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_noopwgcontact_challenger001",
|
| 1700 |
+
"label": "K4 compatible residual retrieval, near-miss global + no-op/wrong-gripper contact challenger",
|
| 1701 |
+
"path": "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_noopwgcontact_challenger0p01_summary.json",
|
| 1702 |
"clean_deployment": "yes",
|
| 1703 |
"same_state_proposals": "no",
|
| 1704 |
"expert_proposal": "no",
|
| 1705 |
+
"story_role": "contact-task sparse-family challenger ablation using no-op and wrong-gripper residuals where oracle trace shows proposal headroom",
|
| 1706 |
"fallback_success": null,
|
| 1707 |
+
"pending_job": "14995045/14995048",
|
| 1708 |
"path_exists": false,
|
| 1709 |
"status": "pending",
|
| 1710 |
"success": null,
|
results/paper_table_status.md
CHANGED
|
@@ -89,8 +89,10 @@ Baseline h=16 policy: 29.74%
|
|
| 89 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgmargin005_challenger001 | K4 compatible residual retrieval, near-miss challenger 0.01, wrong-gripper margin 0.05 | complete | 35.59% | +5.86 pp | yes | no | no | family-specific reliability margin upper setting for wrong-gripper challenger evidence |
|
| 90 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull | K4 compatible residual retrieval, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only | complete | 36.00% | +6.26 pp | yes | no | no | task-conditional challenger gate using branch trace evidence that wrong-gripper tangents help Pick/Pull but harm other tasks |
|
| 91 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001 | K4 compatible residual retrieval, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01 | complete | 36.06% | +6.32 pp | yes | no | no | type-specific affordance gate: keep near-miss global while restricting wrong-gripper challenger to acquisition/contact tasks |
|
| 92 |
-
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin003_challenger001 | K4 compatible residual retrieval, near-miss global + wrong-gripper Pick/Pull margin 0.03 |
|
| 93 |
-
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin005_challenger001 | K4 compatible residual retrieval, near-miss global + wrong-gripper Pick/Pull margin 0.05 |
|
|
|
|
|
|
|
| 94 |
| typed_proposal_lattice_types6_prepend_margin000 | Typed proposal lattice head, six families, policy-prepended margin 0.00 | complete | 31.30% | +1.57 pp | yes | no | no | model-generated counterfactual proposal support test; field scores typed proposals rather than retrieved train-state residuals |
|
| 95 |
| typed_proposal_lattice_types6_prepend_margin005 | Typed proposal lattice head, six families, policy-prepended margin 0.05 | complete | 32.17% | +2.43 pp | yes | no | no | model-generated counterfactual proposal support test with a conservative policy-abstention margin |
|
| 96 |
| typed_proposal_lattice_types4safe_prepend_margin005 | Typed proposal lattice head, safe four families, policy-prepended margin 0.05 | complete | 33.45% | +3.71 pp | yes | no | no | anti-goal proposal-family mask test for the model-generated typed lattice |
|
|
|
|
| 89 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgmargin005_challenger001 | K4 compatible residual retrieval, near-miss challenger 0.01, wrong-gripper margin 0.05 | complete | 35.59% | +5.86 pp | yes | no | no | family-specific reliability margin upper setting for wrong-gripper challenger evidence |
|
| 90 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull | K4 compatible residual retrieval, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only | complete | 36.00% | +6.26 pp | yes | no | no | task-conditional challenger gate using branch trace evidence that wrong-gripper tangents help Pick/Pull but harm other tasks |
|
| 91 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001 | K4 compatible residual retrieval, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01 | complete | 36.06% | +6.32 pp | yes | no | no | type-specific affordance gate: keep near-miss global while restricting wrong-gripper challenger to acquisition/contact tasks |
|
| 92 |
+
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin003_challenger001 | K4 compatible residual retrieval, near-miss global + wrong-gripper Pick/Pull margin 0.03 | complete | 36.00% | +6.26 pp | yes | no | no | type-specific affordance margin: keep near-miss globally sensitive while requiring stronger wrong-gripper evidence only on Pick/Pull |
|
| 93 |
+
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin005_challenger001 | K4 compatible residual retrieval, near-miss global + wrong-gripper Pick/Pull margin 0.05 | complete | 35.88% | +6.14 pp | yes | no | no | upper setting for type-specific wrong-gripper evidence on Pick/Pull while preserving global near-miss calibration |
|
| 94 |
+
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpullstack_challenger001 | K4 compatible residual retrieval, near-miss global + wrong-gripper challenger on Pick/Pull/Stack | pending 14995028/14995047 | pending | pending | yes | no | no | oracle-trace contact-task calibration: wrong-gripper residuals are only challenger-eligible on Pick/Pull/Stack while near-miss stays global |
|
| 95 |
+
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_noopwgcontact_challenger001 | K4 compatible residual retrieval, near-miss global + no-op/wrong-gripper contact challenger | pending 14995045/14995048 | pending | pending | yes | no | no | contact-task sparse-family challenger ablation using no-op and wrong-gripper residuals where oracle trace shows proposal headroom |
|
| 96 |
| typed_proposal_lattice_types6_prepend_margin000 | Typed proposal lattice head, six families, policy-prepended margin 0.00 | complete | 31.30% | +1.57 pp | yes | no | no | model-generated counterfactual proposal support test; field scores typed proposals rather than retrieved train-state residuals |
|
| 97 |
| typed_proposal_lattice_types6_prepend_margin005 | Typed proposal lattice head, six families, policy-prepended margin 0.05 | complete | 32.17% | +2.43 pp | yes | no | no | model-generated counterfactual proposal support test with a conservative policy-abstention margin |
|
| 98 |
| typed_proposal_lattice_types4safe_prepend_margin005 | Typed proposal lattice head, safe four families, policy-prepended margin 0.05 | complete | 33.45% | +3.71 pp | yes | no | no | anti-goal proposal-family mask test for the model-generated typed lattice |
|
scripts/build_paper_analysis.py
CHANGED
|
@@ -513,6 +513,22 @@ METHODS = [
|
|
| 513 |
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_wgmargin0p05_challenger0p01_summary.json"
|
| 514 |
),
|
| 515 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 516 |
MethodSpec(
|
| 517 |
key="typed_proposal_lattice_types6_prepend_margin000",
|
| 518 |
label="Typed proposal lattice head, six families, policy-prepended margin 0.00",
|
|
|
|
| 513 |
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_wgmargin0p05_challenger0p01_summary.json"
|
| 514 |
),
|
| 515 |
),
|
| 516 |
+
MethodSpec(
|
| 517 |
+
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpullstack_challenger001",
|
| 518 |
+
label="K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull/Stack",
|
| 519 |
+
summary_path=(
|
| 520 |
+
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
|
| 521 |
+
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpullstack_challenger0p01_summary.json"
|
| 522 |
+
),
|
| 523 |
+
),
|
| 524 |
+
MethodSpec(
|
| 525 |
+
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_noopwgcontact_challenger001",
|
| 526 |
+
label="K4 compatible tangents, no-op bonus 0.03, near-miss global + no-op/wrong-gripper contact challenger",
|
| 527 |
+
summary_path=(
|
| 528 |
+
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
|
| 529 |
+
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_noopwgcontact_challenger0p01_summary.json"
|
| 530 |
+
),
|
| 531 |
+
),
|
| 532 |
MethodSpec(
|
| 533 |
key="typed_proposal_lattice_types6_prepend_margin000",
|
| 534 |
label="Typed proposal lattice head, six families, policy-prepended margin 0.00",
|
scripts/build_paper_table_status.py
CHANGED
|
@@ -895,6 +895,26 @@ SPECS = [
|
|
| 895 |
story_role="upper setting for type-specific wrong-gripper evidence on Pick/Pull while preserving global near-miss calibration",
|
| 896 |
pending_job="14991123/14991126",
|
| 897 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 898 |
ResultSpec(
|
| 899 |
key="typed_proposal_lattice_types6_prepend_margin000",
|
| 900 |
label="Typed proposal lattice head, six families, policy-prepended margin 0.00",
|
|
|
|
| 895 |
story_role="upper setting for type-specific wrong-gripper evidence on Pick/Pull while preserving global near-miss calibration",
|
| 896 |
pending_job="14991123/14991126",
|
| 897 |
),
|
| 898 |
+
ResultSpec(
|
| 899 |
+
key="retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpullstack_challenger001",
|
| 900 |
+
label="K4 compatible residual retrieval, near-miss global + wrong-gripper challenger on Pick/Pull/Stack",
|
| 901 |
+
path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpullstack_challenger0p01_summary.json",
|
| 902 |
+
clean_deployment="yes",
|
| 903 |
+
same_state_proposals="no",
|
| 904 |
+
expert_proposal="no",
|
| 905 |
+
story_role="oracle-trace contact-task calibration: wrong-gripper residuals are only challenger-eligible on Pick/Pull/Stack while near-miss stays global",
|
| 906 |
+
pending_job="14995028/14995047",
|
| 907 |
+
),
|
| 908 |
+
ResultSpec(
|
| 909 |
+
key="retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_noopwgcontact_challenger001",
|
| 910 |
+
label="K4 compatible residual retrieval, near-miss global + no-op/wrong-gripper contact challenger",
|
| 911 |
+
path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_noopwgcontact_challenger0p01_summary.json",
|
| 912 |
+
clean_deployment="yes",
|
| 913 |
+
same_state_proposals="no",
|
| 914 |
+
expert_proposal="no",
|
| 915 |
+
story_role="contact-task sparse-family challenger ablation using no-op and wrong-gripper residuals where oracle trace shows proposal headroom",
|
| 916 |
+
pending_job="14995045/14995048",
|
| 917 |
+
),
|
| 918 |
ResultSpec(
|
| 919 |
key="typed_proposal_lattice_types6_prepend_margin000",
|
| 920 |
label="Typed proposal lattice head, six families, policy-prepended margin 0.00",
|