Auto-sync: 2026-06-30 02:54:08
Browse files- results/paper_analysis.json +132 -25
- results/paper_analysis.md +18 -11
- results/paper_table_status.json +8 -7
- results/paper_table_status.md +2 -1
- scripts/slurm/summarize_h16_policy_ckpt.sbatch +85 -0
results/paper_analysis.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
| 1 |
{
|
| 2 |
-
"best_clean_key": "
|
| 3 |
-
"generated_utc": "2026-06-30T06:
|
| 4 |
"mechanism_gap": {
|
| 5 |
-
"best_clean_vs_direct_same_ckpt": 0.
|
| 6 |
-
"best_clean_vs_h16": 0.
|
| 7 |
"same_state_full_vs_no_expert": 0.12347826086956515,
|
| 8 |
-
"same_state_no_expert_vs_best_clean": 0.
|
| 9 |
"same_state_no_expert_vs_h16": 0.27246376811594214
|
| 10 |
},
|
| 11 |
"methods": {
|
|
@@ -1433,9 +1433,116 @@
|
|
| 1433 |
"std_success": 0.009889114266221456
|
| 1434 |
},
|
| 1435 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8": {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1436 |
"label": "K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 diagnostic",
|
| 1437 |
-
"
|
| 1438 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1439 |
},
|
| 1440 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_srcscorebonus002": {
|
| 1441 |
"ci95_success": 0.02131295542962397,
|
|
@@ -7097,38 +7204,38 @@
|
|
| 7097 |
},
|
| 7098 |
"paired_deltas": {
|
| 7099 |
"best_clean - canonical_h16": {
|
| 7100 |
-
"ci95_delta": 0.
|
| 7101 |
-
"left": "
|
| 7102 |
-
"mean_delta": 0.
|
| 7103 |
"right": "h16_policy_canonical",
|
| 7104 |
"seed_deltas": {
|
| 7105 |
-
"0": 0.
|
| 7106 |
"1": 0.038260869565217404,
|
| 7107 |
-
"2": 0.
|
| 7108 |
},
|
| 7109 |
"seeds": [
|
| 7110 |
0,
|
| 7111 |
1,
|
| 7112 |
2
|
| 7113 |
],
|
| 7114 |
-
"std_delta": 0.
|
| 7115 |
},
|
| 7116 |
"best_clean - direct_same_ckpt": {
|
| 7117 |
-
"ci95_delta": 0.
|
| 7118 |
-
"left": "
|
| 7119 |
-
"mean_delta": 0.
|
| 7120 |
"right": "near_miss_policy_bc5",
|
| 7121 |
"seed_deltas": {
|
| 7122 |
-
"0": 0.
|
| 7123 |
"1": 0.055652173913043446,
|
| 7124 |
-
"2": 0.
|
| 7125 |
},
|
| 7126 |
"seeds": [
|
| 7127 |
0,
|
| 7128 |
1,
|
| 7129 |
2
|
| 7130 |
],
|
| 7131 |
-
"std_delta": 0.
|
| 7132 |
},
|
| 7133 |
"full_lattice - no_expert_lattice": {
|
| 7134 |
"ci95_delta": 0.02628111194117426,
|
|
@@ -7186,16 +7293,16 @@
|
|
| 7186 |
"best_clean_vs_h16": {
|
| 7187 |
"LiftPegUpright-v1": 0.03278131303915899,
|
| 7188 |
"PickCube-v1": 0.1219595621769535,
|
| 7189 |
-
"PullCube-v1": 0.
|
| 7190 |
-
"PushCube-v1": 0.
|
| 7191 |
-
"StackCube-v1": 0.
|
| 7192 |
},
|
| 7193 |
"no_expert_vs_best_clean": {
|
| 7194 |
"LiftPegUpright-v1": 0.35374851747103375,
|
| 7195 |
"PickCube-v1": 0.29366195398804096,
|
| 7196 |
-
"PullCube-v1": -0.
|
| 7197 |
-
"PushCube-v1": 0.
|
| 7198 |
-
"StackCube-v1": 0.
|
| 7199 |
}
|
| 7200 |
}
|
| 7201 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"best_clean_key": "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8",
|
| 3 |
+
"generated_utc": "2026-06-30T06:57:02+00:00",
|
| 4 |
"mechanism_gap": {
|
| 5 |
+
"best_clean_vs_direct_same_ckpt": 0.07362318840579707,
|
| 6 |
+
"best_clean_vs_h16": 0.05913043478260871,
|
| 7 |
"same_state_full_vs_no_expert": 0.12347826086956515,
|
| 8 |
+
"same_state_no_expert_vs_best_clean": 0.21333333333333343,
|
| 9 |
"same_state_no_expert_vs_h16": 0.27246376811594214
|
| 10 |
},
|
| 11 |
"methods": {
|
|
|
|
| 1433 |
"std_success": 0.009889114266221456
|
| 1434 |
},
|
| 1435 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8": {
|
| 1436 |
+
"candidate_oracle_rollouts": 8,
|
| 1437 |
+
"candidate_oracle_type_counts": {
|
| 1438 |
+
"retrieval_residual_policy_residual": 867,
|
| 1439 |
+
"retrieval_residual_residual_near_miss": 201,
|
| 1440 |
+
"retrieval_residual_residual_near_miss+residual_wrong_gripper": 99,
|
| 1441 |
+
"retrieval_residual_residual_no_op": 182,
|
| 1442 |
+
"retrieval_residual_residual_no_op+residual_wrong_gripper": 86,
|
| 1443 |
+
"retrieval_residual_residual_wrong_gripper": 290
|
| 1444 |
+
},
|
| 1445 |
+
"candidate_oracle_unique_tolerance": 1e-06,
|
| 1446 |
+
"ci95_success": 0.03024411731871433,
|
| 1447 |
"label": "K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 diagnostic",
|
| 1448 |
+
"mean_action_mse_to_best": 0.4063519326314006,
|
| 1449 |
+
"mean_candidate_oracle_improvement_rate": 0.4602898550724637,
|
| 1450 |
+
"mean_candidate_oracle_progress": 0.6443418554713328,
|
| 1451 |
+
"mean_candidate_oracle_score_gain_over_selected": 0.16034898876064066,
|
| 1452 |
+
"mean_candidate_oracle_success_rate": 0.4307246376811594,
|
| 1453 |
+
"mean_candidate_oracle_unique_count": NaN,
|
| 1454 |
+
"mean_progress": 0.571575072284851,
|
| 1455 |
+
"mean_success": 0.3565217391304348,
|
| 1456 |
+
"num_completed": 3,
|
| 1457 |
+
"per_task_success": {
|
| 1458 |
+
"LiftPegUpright-v1": {
|
| 1459 |
+
"mean_num_groups": 102.0,
|
| 1460 |
+
"mean_success": 0.26340865087329823,
|
| 1461 |
+
"std_success": 0.06057881385012102
|
| 1462 |
+
},
|
| 1463 |
+
"PickCube-v1": {
|
| 1464 |
+
"mean_num_groups": 196.66666666666666,
|
| 1465 |
+
"mean_success": 0.3278169093386485,
|
| 1466 |
+
"std_success": 0.02533117351097172
|
| 1467 |
+
},
|
| 1468 |
+
"PullCube-v1": {
|
| 1469 |
+
"mean_num_groups": 81.0,
|
| 1470 |
+
"mean_success": 0.21716032505506191,
|
| 1471 |
+
"std_success": 0.0182535211270701
|
| 1472 |
+
},
|
| 1473 |
+
"PushCube-v1": {
|
| 1474 |
+
"mean_num_groups": 102.0,
|
| 1475 |
+
"mean_success": 0.7541548289389031,
|
| 1476 |
+
"std_success": 0.08244271787813129
|
| 1477 |
+
},
|
| 1478 |
+
"StackCube-v1": {
|
| 1479 |
+
"mean_num_groups": 93.33333333333333,
|
| 1480 |
+
"mean_success": 0.19707329707329704,
|
| 1481 |
+
"std_success": 0.0831467196651622
|
| 1482 |
+
}
|
| 1483 |
+
},
|
| 1484 |
+
"seed_action_mse_to_best": {
|
| 1485 |
+
"0": 0.3930192190424904,
|
| 1486 |
+
"1": 0.39824521353549286,
|
| 1487 |
+
"2": 0.42779136531621864
|
| 1488 |
+
},
|
| 1489 |
+
"seed_progress": {
|
| 1490 |
+
"0": 0.5579317773906919,
|
| 1491 |
+
"1": 0.571580135051323,
|
| 1492 |
+
"2": 0.5852133044125383
|
| 1493 |
+
},
|
| 1494 |
+
"seed_success": {
|
| 1495 |
+
"0": 0.35130434782608694,
|
| 1496 |
+
"1": 0.34782608695652173,
|
| 1497 |
+
"2": 0.37043478260869567
|
| 1498 |
+
},
|
| 1499 |
+
"selected_candidate_type_counts": {
|
| 1500 |
+
"retrieval_residual_policy_residual": 1613,
|
| 1501 |
+
"retrieval_residual_residual_near_miss": 8,
|
| 1502 |
+
"retrieval_residual_residual_near_miss+residual_wrong_gripper": 5,
|
| 1503 |
+
"retrieval_residual_residual_no_op": 56,
|
| 1504 |
+
"retrieval_residual_residual_no_op+residual_wrong_gripper": 24,
|
| 1505 |
+
"retrieval_residual_residual_wrong_gripper": 19
|
| 1506 |
+
},
|
| 1507 |
+
"selected_residual_scale_counts": {
|
| 1508 |
+
"0.35": 1631,
|
| 1509 |
+
"0.4": 11,
|
| 1510 |
+
"0.45": 83
|
| 1511 |
+
},
|
| 1512 |
+
"selected_type_outcomes": {
|
| 1513 |
+
"retrieval_residual_policy_residual": {
|
| 1514 |
+
"count": 1613.0,
|
| 1515 |
+
"mean_progress": 0.561925975703087,
|
| 1516 |
+
"success_rate": 0.34593924364538126
|
| 1517 |
+
},
|
| 1518 |
+
"retrieval_residual_residual_near_miss": {
|
| 1519 |
+
"count": 8.0,
|
| 1520 |
+
"mean_progress": 0.8754628524184227,
|
| 1521 |
+
"success_rate": 0.75
|
| 1522 |
+
},
|
| 1523 |
+
"retrieval_residual_residual_near_miss+residual_wrong_gripper": {
|
| 1524 |
+
"count": 5.0,
|
| 1525 |
+
"mean_progress": 0.7557869911193847,
|
| 1526 |
+
"success_rate": 0.6
|
| 1527 |
+
},
|
| 1528 |
+
"retrieval_residual_residual_no_op": {
|
| 1529 |
+
"count": 56.0,
|
| 1530 |
+
"mean_progress": 0.7249722566921264,
|
| 1531 |
+
"success_rate": 0.5
|
| 1532 |
+
},
|
| 1533 |
+
"retrieval_residual_residual_no_op+residual_wrong_gripper": {
|
| 1534 |
+
"count": 24.0,
|
| 1535 |
+
"mean_progress": 0.6711457151298722,
|
| 1536 |
+
"success_rate": 0.4583333333333333
|
| 1537 |
+
},
|
| 1538 |
+
"retrieval_residual_residual_wrong_gripper": {
|
| 1539 |
+
"count": 19.0,
|
| 1540 |
+
"mean_progress": 0.6364115562878156,
|
| 1541 |
+
"success_rate": 0.47368421052631576
|
| 1542 |
+
}
|
| 1543 |
+
},
|
| 1544 |
+
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_oraclek8_summary.json",
|
| 1545 |
+
"std_success": 0.012173913043478276
|
| 1546 |
},
|
| 1547 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_srcscorebonus002": {
|
| 1548 |
"ci95_success": 0.02131295542962397,
|
|
|
|
| 7204 |
},
|
| 7205 |
"paired_deltas": {
|
| 7206 |
"best_clean - canonical_h16": {
|
| 7207 |
+
"ci95_delta": 0.04510826498554671,
|
| 7208 |
+
"left": "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8",
|
| 7209 |
+
"mean_delta": 0.05913043478260869,
|
| 7210 |
"right": "h16_policy_canonical",
|
| 7211 |
"seed_deltas": {
|
| 7212 |
+
"0": 0.0678260869565217,
|
| 7213 |
"1": 0.038260869565217404,
|
| 7214 |
+
"2": 0.07130434782608697
|
| 7215 |
},
|
| 7216 |
"seeds": [
|
| 7217 |
0,
|
| 7218 |
1,
|
| 7219 |
2
|
| 7220 |
],
|
| 7221 |
+
"std_delta": 0.018157054798105293
|
| 7222 |
},
|
| 7223 |
"best_clean - direct_same_ckpt": {
|
| 7224 |
+
"ci95_delta": 0.045383317930687055,
|
| 7225 |
+
"left": "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8",
|
| 7226 |
+
"mean_delta": 0.07362318840579708,
|
| 7227 |
"right": "near_miss_policy_bc5",
|
| 7228 |
"seed_deltas": {
|
| 7229 |
+
"0": 0.07304347826086954,
|
| 7230 |
"1": 0.055652173913043446,
|
| 7231 |
+
"2": 0.09217391304347827
|
| 7232 |
},
|
| 7233 |
"seeds": [
|
| 7234 |
0,
|
| 7235 |
1,
|
| 7236 |
2
|
| 7237 |
],
|
| 7238 |
+
"std_delta": 0.018267769572856522
|
| 7239 |
},
|
| 7240 |
"full_lattice - no_expert_lattice": {
|
| 7241 |
"ci95_delta": 0.02628111194117426,
|
|
|
|
| 7293 |
"best_clean_vs_h16": {
|
| 7294 |
"LiftPegUpright-v1": 0.03278131303915899,
|
| 7295 |
"PickCube-v1": 0.1219595621769535,
|
| 7296 |
+
"PullCube-v1": 0.019305840358471965,
|
| 7297 |
+
"PushCube-v1": 0.003597657063003523,
|
| 7298 |
+
"StackCube-v1": 0.05610685610685609
|
| 7299 |
},
|
| 7300 |
"no_expert_vs_best_clean": {
|
| 7301 |
"LiftPegUpright-v1": 0.35374851747103375,
|
| 7302 |
"PickCube-v1": 0.29366195398804096,
|
| 7303 |
+
"PullCube-v1": -0.018680539733171325,
|
| 7304 |
+
"PushCube-v1": 0.04688815632569587,
|
| 7305 |
+
"StackCube-v1": 0.2854441854441855
|
| 7306 |
}
|
| 7307 |
}
|
| 7308 |
}
|
results/paper_analysis.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
# Paper Analysis
|
| 2 |
|
| 3 |
-
Generated: `2026-06-30T06:
|
| 4 |
|
| 5 |
## Main Seed Statistics
|
| 6 |
|
|
@@ -50,7 +50,7 @@ Generated: `2026-06-30T06:41:21+00:00`
|
|
| 50 |
| residual_k4_composemasked_dropnmnoop_grid035040045 | K4 composed type-consensus tangents, masked, drop near-miss+no-op composite | 3 | 35.48% +/- 1.25 | +/- 3.12 | 57.00% | 0.406 | +5.74 pp |
|
| 51 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003 | K4 composed type-consensus tangents, masked, drop near-miss+no-op composite | 3 | 35.59% +/- 0.99 | +/- 2.46 | 57.07% | 0.406 | +5.86 pp |
|
| 52 |
| residual_k4_composemasked_dropnmnoop_grid035040045_margin010_noopbonus003 | K4 compatible tangents, margin 0.10, no-op bonus 0.03 | 3 | 34.67% +/- 0.96 | +/- 2.38 | 56.45% | 0.418 | +4.93 pp |
|
| 53 |
-
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8 | K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 diagnostic |
|
| 54 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001 | K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.01 | 3 | 35.59% +/- 0.99 | +/- 2.46 | 57.10% | 0.406 | +5.86 pp |
|
| 55 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus002 | K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.02 | 3 | 35.59% +/- 0.99 | +/- 2.46 | 57.10% | 0.406 | +5.86 pp |
|
| 56 |
| residual_k4_composemasked_dropnmnoop_grid035040045_srcscorebonus002 | K4 composed type-consensus tangents, masked, drop near-miss+no-op composite, source-score bonus 0.02 | 3 | 35.48% +/- 1.22 | +/- 3.02 | 57.02% | 0.408 | +5.74 pp |
|
|
@@ -98,8 +98,8 @@ Generated: `2026-06-30T06:41:21+00:00`
|
|
| 98 |
|
| 99 |
| comparison | seeds | mean delta | 95% CI | seed deltas |
|
| 100 |
|---|---:|---:|---:|---|
|
| 101 |
-
| best_clean - canonical_h16 | 3 | +5.
|
| 102 |
-
| best_clean - direct_same_ckpt | 3 | +7.
|
| 103 |
| no_expert_lattice - canonical_h16 | 3 | +27.25 pp | +/- 8.58 | 0:+23.30, 1:+28.70, 2:+29.74 |
|
| 104 |
| full_lattice - no_expert_lattice | 3 | +12.35 pp | +/- 2.63 | 0:+13.57, 1:+11.83, 2:+11.65 |
|
| 105 |
| policy_candidate_lattice - no_expert_lattice | 3 | -16.29 pp | +/- 7.55 | 0:-15.48, 1:-13.74, 2:-19.65 |
|
|
@@ -110,25 +110,32 @@ Generated: `2026-06-30T06:41:21+00:00`
|
|
| 110 |
|---|---:|---:|---:|---:|---:|---:|---:|
|
| 111 |
| LiftPegUpright-v1 | 23.06% | 26.34% | 62.90% | 61.72% | 73.19% | +3.28 pp | +35.37 pp |
|
| 112 |
| PickCube-v1 | 20.59% | 32.78% | 58.13% | 62.15% | 84.19% | +12.20 pp | +29.37 pp |
|
| 113 |
-
| PullCube-v1 | 19.79% |
|
| 114 |
-
| PushCube-v1 | 75.06% | 75.
|
| 115 |
-
| StackCube-v1 | 14.10% |
|
| 116 |
|
| 117 |
## Mechanism Gap
|
| 118 |
|
| 119 |
-
- Best clean residual transport improves over canonical h16 by +5.
|
| 120 |
- Same-state no-expert lattice improves over canonical h16 by +27.25 pp.
|
| 121 |
-
- Remaining clean-to-same-state proposal gap is +21.
|
| 122 |
- Full lattice adds expert proposals and reaches 69.33%, a +12.35 pp gain over no-expert.
|
| 123 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
## Selection Histograms
|
| 125 |
|
| 126 |
- `same_state_near_miss`: lattice_near_miss=1725 (100.0%)
|
| 127 |
- `same_state_no_expert`: lattice_near_miss=1263 (73.2%), lattice_no_op=222 (12.9%), lattice_random_negative=144 (8.3%), lattice_wrong_gripper=62 (3.6%), lattice_wrong_direction=34 (2.0%)
|
| 128 |
- `same_state_policy_baseline`: policy_continuous=1022 (59.2%), lattice_near_miss=448 (26.0%), lattice_no_op=119 (6.9%), lattice_random_negative=75 (4.3%), lattice_wrong_gripper=45 (2.6%), lattice_wrong_direction=16 (0.9%)
|
| 129 |
- `same_state_full`: lattice_expert=977 (56.6%), lattice_near_miss=348 (20.2%), lattice_no_op=177 (10.3%), lattice_random_negative=138 (8.0%), lattice_wrong_gripper=55 (3.2%), lattice_wrong_direction=30 (1.7%)
|
| 130 |
-
- `
|
| 131 |
-
- `
|
| 132 |
|
| 133 |
## Selected-Type Outcomes
|
| 134 |
|
|
|
|
| 1 |
# Paper Analysis
|
| 2 |
|
| 3 |
+
Generated: `2026-06-30T06:57:02+00:00`
|
| 4 |
|
| 5 |
## Main Seed Statistics
|
| 6 |
|
|
|
|
| 50 |
| residual_k4_composemasked_dropnmnoop_grid035040045 | K4 composed type-consensus tangents, masked, drop near-miss+no-op composite | 3 | 35.48% +/- 1.25 | +/- 3.12 | 57.00% | 0.406 | +5.74 pp |
|
| 51 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003 | K4 composed type-consensus tangents, masked, drop near-miss+no-op composite | 3 | 35.59% +/- 0.99 | +/- 2.46 | 57.07% | 0.406 | +5.86 pp |
|
| 52 |
| residual_k4_composemasked_dropnmnoop_grid035040045_margin010_noopbonus003 | K4 compatible tangents, margin 0.10, no-op bonus 0.03 | 3 | 34.67% +/- 0.96 | +/- 2.38 | 56.45% | 0.418 | +4.93 pp |
|
| 53 |
+
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8 | K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 diagnostic | 3 | 35.65% +/- 1.22 | +/- 3.02 | 57.16% | 0.406 | +5.91 pp |
|
| 54 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001 | K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.01 | 3 | 35.59% +/- 0.99 | +/- 2.46 | 57.10% | 0.406 | +5.86 pp |
|
| 55 |
| residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus002 | K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.02 | 3 | 35.59% +/- 0.99 | +/- 2.46 | 57.10% | 0.406 | +5.86 pp |
|
| 56 |
| residual_k4_composemasked_dropnmnoop_grid035040045_srcscorebonus002 | K4 composed type-consensus tangents, masked, drop near-miss+no-op composite, source-score bonus 0.02 | 3 | 35.48% +/- 1.22 | +/- 3.02 | 57.02% | 0.408 | +5.74 pp |
|
|
|
|
| 98 |
|
| 99 |
| comparison | seeds | mean delta | 95% CI | seed deltas |
|
| 100 |
|---|---:|---:|---:|---|
|
| 101 |
+
| best_clean - canonical_h16 | 3 | +5.91 pp | +/- 4.51 | 0:+6.78, 1:+3.83, 2:+7.13 |
|
| 102 |
+
| best_clean - direct_same_ckpt | 3 | +7.36 pp | +/- 4.54 | 0:+7.30, 1:+5.57, 2:+9.22 |
|
| 103 |
| no_expert_lattice - canonical_h16 | 3 | +27.25 pp | +/- 8.58 | 0:+23.30, 1:+28.70, 2:+29.74 |
|
| 104 |
| full_lattice - no_expert_lattice | 3 | +12.35 pp | +/- 2.63 | 0:+13.57, 1:+11.83, 2:+11.65 |
|
| 105 |
| policy_candidate_lattice - no_expert_lattice | 3 | -16.29 pp | +/- 7.55 | 0:-15.48, 1:-13.74, 2:-19.65 |
|
|
|
|
| 110 |
|---|---:|---:|---:|---:|---:|---:|---:|
|
| 111 |
| LiftPegUpright-v1 | 23.06% | 26.34% | 62.90% | 61.72% | 73.19% | +3.28 pp | +35.37 pp |
|
| 112 |
| PickCube-v1 | 20.59% | 32.78% | 58.13% | 62.15% | 84.19% | +12.20 pp | +29.37 pp |
|
| 113 |
+
| PullCube-v1 | 19.79% | 21.72% | 15.02% | 19.85% | 22.41% | +1.93 pp | -1.87 pp |
|
| 114 |
+
| PushCube-v1 | 75.06% | 75.42% | 82.54% | 80.10% | 81.92% | +0.36 pp | +4.69 pp |
|
| 115 |
+
| StackCube-v1 | 14.10% | 19.71% | 50.41% | 48.25% | 60.83% | +5.61 pp | +28.54 pp |
|
| 116 |
|
| 117 |
## Mechanism Gap
|
| 118 |
|
| 119 |
+
- Best clean residual transport improves over canonical h16 by +5.91 pp.
|
| 120 |
- Same-state no-expert lattice improves over canonical h16 by +27.25 pp.
|
| 121 |
+
- Remaining clean-to-same-state proposal gap is +21.33 pp.
|
| 122 |
- Full lattice adds expert proposals and reaches 69.33%, a +12.35 pp gain over no-expert.
|
| 123 |
|
| 124 |
+
## Candidate-Oracle Diagnostic
|
| 125 |
+
|
| 126 |
+
- Oracle over the deployed candidate prefix reaches 43.07% with mean progress 64.43%; this is diagnostic-only because it uses measured rollout outcomes after generating candidates.
|
| 127 |
+
- Mean oracle-prefix score gain over the selected branch is +0.160, which isolates ranking/abstention headroom inside the clean proposal set.
|
| 128 |
+
- Mean unique candidates in the prefix: nan.
|
| 129 |
+
- Candidate-oracle best type counts: {'retrieval_residual_policy_residual': 867, 'retrieval_residual_residual_near_miss': 201, 'retrieval_residual_residual_near_miss+residual_wrong_gripper': 99, 'retrieval_residual_residual_no_op': 182, 'retrieval_residual_residual_no_op+residual_wrong_gripper': 86, 'retrieval_residual_residual_wrong_gripper': 290}.
|
| 130 |
+
|
| 131 |
## Selection Histograms
|
| 132 |
|
| 133 |
- `same_state_near_miss`: lattice_near_miss=1725 (100.0%)
|
| 134 |
- `same_state_no_expert`: lattice_near_miss=1263 (73.2%), lattice_no_op=222 (12.9%), lattice_random_negative=144 (8.3%), lattice_wrong_gripper=62 (3.6%), lattice_wrong_direction=34 (2.0%)
|
| 135 |
- `same_state_policy_baseline`: policy_continuous=1022 (59.2%), lattice_near_miss=448 (26.0%), lattice_no_op=119 (6.9%), lattice_random_negative=75 (4.3%), lattice_wrong_gripper=45 (2.6%), lattice_wrong_direction=16 (0.9%)
|
| 136 |
- `same_state_full`: lattice_expert=977 (56.6%), lattice_near_miss=348 (20.2%), lattice_no_op=177 (10.3%), lattice_random_negative=138 (8.0%), lattice_wrong_gripper=55 (3.2%), lattice_wrong_direction=30 (1.7%)
|
| 137 |
+
- `residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8`: retrieval_residual_policy_residual=1613 (93.5%), retrieval_residual_residual_no_op=56 (3.2%), retrieval_residual_residual_no_op+residual_wrong_gripper=24 (1.4%), retrieval_residual_residual_wrong_gripper=19 (1.1%), retrieval_residual_residual_near_miss=8 (0.5%), retrieval_residual_residual_near_miss+residual_wrong_gripper=5 (0.3%)
|
| 138 |
+
- `residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8` residual scale counts: {'0.35': 1631, '0.4': 11, '0.45': 83}
|
| 139 |
|
| 140 |
## Selected-Type Outcomes
|
| 141 |
|
results/paper_table_status.json
CHANGED
|
@@ -1363,14 +1363,14 @@
|
|
| 1363 |
"story_role": "diagnostic oracle over generated clean candidate prefix to separate proposal headroom from field ranking",
|
| 1364 |
"fallback_success": null,
|
| 1365 |
"pending_job": "14953513/14953522",
|
| 1366 |
-
"path_exists":
|
| 1367 |
-
"status": "
|
| 1368 |
-
"success":
|
| 1369 |
-
"std_success":
|
| 1370 |
"completed_seeds": null,
|
| 1371 |
-
"num_completed":
|
| 1372 |
"best_config": null,
|
| 1373 |
-
"gain_vs_h16_policy":
|
| 1374 |
},
|
| 1375 |
{
|
| 1376 |
"key": "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001",
|
|
@@ -2618,6 +2618,7 @@
|
|
| 2618 |
"KNN counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.",
|
| 2619 |
"K4 train-state residual retrieval, scale 0.40, safe residuals, mean-by-type tangent consensus improves the clean bridge but is not yet the main result.",
|
| 2620 |
"Policy-baseline abstention inside the same-state no-expert lattice drops from 56.99% to 40.70%, so the mechanism result should emphasize counterfactual proposal geometry rather than policy fallback.",
|
| 2621 |
-
"Singleton near-miss priors 0.01/0.02 tie the clean best rather than improving it; the compatible chart preserves high-precision near-miss tangents, but they remain sparse."
|
|
|
|
| 2622 |
]
|
| 2623 |
}
|
|
|
|
| 1363 |
"story_role": "diagnostic oracle over generated clean candidate prefix to separate proposal headroom from field ranking",
|
| 1364 |
"fallback_success": null,
|
| 1365 |
"pending_job": "14953513/14953522",
|
| 1366 |
+
"path_exists": true,
|
| 1367 |
+
"status": "complete",
|
| 1368 |
+
"success": 0.43072463768115943,
|
| 1369 |
+
"std_success": 0.012173913043478276,
|
| 1370 |
"completed_seeds": null,
|
| 1371 |
+
"num_completed": 3,
|
| 1372 |
"best_config": null,
|
| 1373 |
+
"gain_vs_h16_policy": 0.13333333333333336
|
| 1374 |
},
|
| 1375 |
{
|
| 1376 |
"key": "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001",
|
|
|
|
| 2618 |
"KNN counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.",
|
| 2619 |
"K4 train-state residual retrieval, scale 0.40, safe residuals, mean-by-type tangent consensus improves the clean bridge but is not yet the main result.",
|
| 2620 |
"Policy-baseline abstention inside the same-state no-expert lattice drops from 56.99% to 40.70%, so the mechanism result should emphasize counterfactual proposal geometry rather than policy fallback.",
|
| 2621 |
+
"Singleton near-miss priors 0.01/0.02 tie the clean best rather than improving it; the compatible chart preserves high-precision near-miss tangents, but they remain sparse.",
|
| 2622 |
+
"Candidate-oracle prefix K=8 is diagnostic-only; compare it to the clean selected row to decide whether the next paper move should be proposal generation or field ranking."
|
| 2623 |
]
|
| 2624 |
}
|
results/paper_table_status.md
CHANGED
|
@@ -74,7 +74,7 @@ Baseline h=16 policy: 29.74%
|
|
| 74 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045 | K4 composed type-consensus residual retrieval, masked, drop near-miss+no-op composite | complete | 35.48% | +5.74 pp | yes | no | no | exact compatibility mask without typed no-op prior |
|
| 75 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003 | K4 composed type-consensus residual retrieval, masked, drop near-miss+no-op composite | complete | 35.59% | +5.86 pp | yes | no | no | exact incompatibility mask for one weak composed tangent pair |
|
| 76 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_margin010_noopbonus003 | K4 compatible residual retrieval, margin 0.10, no-op bonus 0.03 | complete | 34.67% | +4.93 pp | yes | no | no | oracle-motivated abstention sensitivity on the compatible local tangent chart |
|
| 77 |
-
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8 | K4 compatible residual retrieval, unique candidate-oracle prefix K=8 diagnostic |
|
| 78 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001 | K4 composed compatible residual retrieval, no-op bonus 0.03, singleton near-miss bonus 0.01 | complete | 35.59% | +5.86 pp | yes | no | no | revive high-precision singleton near-miss tangents without boosting toxic near-miss+no-op composites |
|
| 79 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus002 | K4 composed compatible residual retrieval, no-op bonus 0.03, singleton near-miss bonus 0.02 | complete | 35.59% | +5.86 pp | yes | no | no | stronger singleton near-miss revival prior on the compatible local tangent chart |
|
| 80 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_srcscorebonus002 | K4 composed type-consensus residual retrieval, masked, drop near-miss+no-op composite, source-score bonus 0.02 | complete | 35.48% | +5.74 pp | yes | no | no | train-measured source-score prior on the compatible local tangent chart |
|
|
@@ -151,3 +151,4 @@ Baseline h=16 policy: 29.74%
|
|
| 151 |
- K4 train-state residual retrieval, scale 0.40, safe residuals, mean-by-type tangent consensus improves the clean bridge but is not yet the main result.
|
| 152 |
- Policy-baseline abstention inside the same-state no-expert lattice drops from 56.99% to 40.70%, so the mechanism result should emphasize counterfactual proposal geometry rather than policy fallback.
|
| 153 |
- Singleton near-miss priors 0.01/0.02 tie the clean best rather than improving it; the compatible chart preserves high-precision near-miss tangents, but they remain sparse.
|
|
|
|
|
|
| 74 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045 | K4 composed type-consensus residual retrieval, masked, drop near-miss+no-op composite | complete | 35.48% | +5.74 pp | yes | no | no | exact compatibility mask without typed no-op prior |
|
| 75 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003 | K4 composed type-consensus residual retrieval, masked, drop near-miss+no-op composite | complete | 35.59% | +5.86 pp | yes | no | no | exact incompatibility mask for one weak composed tangent pair |
|
| 76 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_margin010_noopbonus003 | K4 compatible residual retrieval, margin 0.10, no-op bonus 0.03 | complete | 34.67% | +4.93 pp | yes | no | no | oracle-motivated abstention sensitivity on the compatible local tangent chart |
|
| 77 |
+
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8 | K4 compatible residual retrieval, unique candidate-oracle prefix K=8 diagnostic | complete | 43.07% | +13.33 pp | diagnostic | no | no | diagnostic oracle over generated clean candidate prefix to separate proposal headroom from field ranking |
|
| 78 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001 | K4 composed compatible residual retrieval, no-op bonus 0.03, singleton near-miss bonus 0.01 | complete | 35.59% | +5.86 pp | yes | no | no | revive high-precision singleton near-miss tangents without boosting toxic near-miss+no-op composites |
|
| 79 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus002 | K4 composed compatible residual retrieval, no-op bonus 0.03, singleton near-miss bonus 0.02 | complete | 35.59% | +5.86 pp | yes | no | no | stronger singleton near-miss revival prior on the compatible local tangent chart |
|
| 80 |
| retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_srcscorebonus002 | K4 composed type-consensus residual retrieval, masked, drop near-miss+no-op composite, source-score bonus 0.02 | complete | 35.48% | +5.74 pp | yes | no | no | train-measured source-score prior on the compatible local tangent chart |
|
|
|
|
| 151 |
- K4 train-state residual retrieval, scale 0.40, safe residuals, mean-by-type tangent consensus improves the clean bridge but is not yet the main result.
|
| 152 |
- Policy-baseline abstention inside the same-state no-expert lattice drops from 56.99% to 40.70%, so the mechanism result should emphasize counterfactual proposal geometry rather than policy fallback.
|
| 153 |
- Singleton near-miss priors 0.01/0.02 tie the clean best rather than improving it; the compatible chart preserves high-precision near-miss tangents, but they remain sparse.
|
| 154 |
+
- Candidate-oracle prefix K=8 is diagnostic-only; compare it to the clean selected row to decide whether the next paper move should be proposal generation or field ranking.
|
scripts/slurm/summarize_h16_policy_ckpt.sbatch
CHANGED
|
@@ -132,6 +132,21 @@ for result_path in sorted(base_dir.glob(f"seed_*/{out_name}")):
|
|
| 132 |
"candidate_oracle_type_counts": data.get(
|
| 133 |
"candidate_oracle_type_counts", {}
|
| 134 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
"selected_residual_scale_counts": dict(selected_scale_counts),
|
| 136 |
"policy_rollout_success_rate": data.get("policy_rollout_success_rate", 0.0),
|
| 137 |
"policy_rollout_progress": data.get("policy_rollout_progress", 0.0),
|
|
@@ -168,6 +183,21 @@ summary = {
|
|
| 168 |
"rows": rows,
|
| 169 |
}
|
| 170 |
if candidate_oracle_rows:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
summary.update(
|
| 172 |
{
|
| 173 |
"candidate_oracle_rollouts": max(
|
|
@@ -225,6 +255,34 @@ if candidate_oracle_rows:
|
|
| 225 |
Counter(),
|
| 226 |
)
|
| 227 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 228 |
}
|
| 229 |
)
|
| 230 |
|
|
@@ -286,6 +344,33 @@ if candidate_oracle_rows:
|
|
| 286 |
),
|
| 287 |
]
|
| 288 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 289 |
lines.extend(
|
| 290 |
[
|
| 291 |
"",
|
|
|
|
| 132 |
"candidate_oracle_type_counts": data.get(
|
| 133 |
"candidate_oracle_type_counts", {}
|
| 134 |
),
|
| 135 |
+
"candidate_oracle_best_branch_rank": data.get(
|
| 136 |
+
"candidate_oracle_best_branch_rank"
|
| 137 |
+
),
|
| 138 |
+
"candidate_oracle_best_branch_rank_counts": data.get(
|
| 139 |
+
"candidate_oracle_best_branch_rank_counts", {}
|
| 140 |
+
),
|
| 141 |
+
"candidate_oracle_branch_success_rates": data.get(
|
| 142 |
+
"candidate_oracle_branch_success_rates", []
|
| 143 |
+
),
|
| 144 |
+
"candidate_oracle_branch_progress": data.get(
|
| 145 |
+
"candidate_oracle_branch_progress", []
|
| 146 |
+
),
|
| 147 |
+
"candidate_oracle_branch_score_gains_over_selected": data.get(
|
| 148 |
+
"candidate_oracle_branch_score_gains_over_selected", []
|
| 149 |
+
),
|
| 150 |
"selected_residual_scale_counts": dict(selected_scale_counts),
|
| 151 |
"policy_rollout_success_rate": data.get("policy_rollout_success_rate", 0.0),
|
| 152 |
"policy_rollout_progress": data.get("policy_rollout_progress", 0.0),
|
|
|
|
| 183 |
"rows": rows,
|
| 184 |
}
|
| 185 |
if candidate_oracle_rows:
|
| 186 |
+
def mean_branch_metric(key: str) -> list[float]:
|
| 187 |
+
max_len = max(
|
| 188 |
+
(len(row.get(key) or []) for row in candidate_oracle_rows),
|
| 189 |
+
default=0,
|
| 190 |
+
)
|
| 191 |
+
values = []
|
| 192 |
+
for branch_index in range(max_len):
|
| 193 |
+
branch_values = [
|
| 194 |
+
row[key][branch_index]
|
| 195 |
+
for row in candidate_oracle_rows
|
| 196 |
+
if len(row.get(key) or []) > branch_index
|
| 197 |
+
]
|
| 198 |
+
values.append(statistics.mean(branch_values) if branch_values else 0.0)
|
| 199 |
+
return values
|
| 200 |
+
|
| 201 |
summary.update(
|
| 202 |
{
|
| 203 |
"candidate_oracle_rollouts": max(
|
|
|
|
| 255 |
Counter(),
|
| 256 |
)
|
| 257 |
),
|
| 258 |
+
"mean_candidate_oracle_best_branch_rank": statistics.mean(
|
| 259 |
+
row["candidate_oracle_best_branch_rank"]
|
| 260 |
+
for row in candidate_oracle_rows
|
| 261 |
+
if row.get("candidate_oracle_best_branch_rank") is not None
|
| 262 |
+
)
|
| 263 |
+
if any(
|
| 264 |
+
row.get("candidate_oracle_best_branch_rank") is not None
|
| 265 |
+
for row in candidate_oracle_rows
|
| 266 |
+
)
|
| 267 |
+
else None,
|
| 268 |
+
"candidate_oracle_best_branch_rank_counts": dict(
|
| 269 |
+
sum(
|
| 270 |
+
(
|
| 271 |
+
Counter(row.get("candidate_oracle_best_branch_rank_counts", {}))
|
| 272 |
+
for row in candidate_oracle_rows
|
| 273 |
+
),
|
| 274 |
+
Counter(),
|
| 275 |
+
)
|
| 276 |
+
),
|
| 277 |
+
"mean_candidate_oracle_branch_success_rates": mean_branch_metric(
|
| 278 |
+
"candidate_oracle_branch_success_rates"
|
| 279 |
+
),
|
| 280 |
+
"mean_candidate_oracle_branch_progress": mean_branch_metric(
|
| 281 |
+
"candidate_oracle_branch_progress"
|
| 282 |
+
),
|
| 283 |
+
"mean_candidate_oracle_branch_score_gains_over_selected": (
|
| 284 |
+
mean_branch_metric("candidate_oracle_branch_score_gains_over_selected")
|
| 285 |
+
),
|
| 286 |
}
|
| 287 |
)
|
| 288 |
|
|
|
|
| 344 |
),
|
| 345 |
]
|
| 346 |
)
|
| 347 |
+
if summary.get("mean_candidate_oracle_best_branch_rank") is not None:
|
| 348 |
+
branch_success = ", ".join(
|
| 349 |
+
f"{value:.2%}"
|
| 350 |
+
for value in summary.get(
|
| 351 |
+
"mean_candidate_oracle_branch_success_rates", []
|
| 352 |
+
)
|
| 353 |
+
)
|
| 354 |
+
branch_gains = ", ".join(
|
| 355 |
+
f"{value:+.3f}"
|
| 356 |
+
for value in summary.get(
|
| 357 |
+
"mean_candidate_oracle_branch_score_gains_over_selected", []
|
| 358 |
+
)
|
| 359 |
+
)
|
| 360 |
+
lines.extend(
|
| 361 |
+
[
|
| 362 |
+
(
|
| 363 |
+
"Candidate-oracle best branch rank: "
|
| 364 |
+
f"{summary['mean_candidate_oracle_best_branch_rank']:.2f}"
|
| 365 |
+
),
|
| 366 |
+
(
|
| 367 |
+
"Candidate-oracle best branch rank counts: "
|
| 368 |
+
f"{summary['candidate_oracle_best_branch_rank_counts']}"
|
| 369 |
+
),
|
| 370 |
+
f"Candidate-oracle branch success by rank: {branch_success}",
|
| 371 |
+
f"Candidate-oracle branch gain by rank: {branch_gains}",
|
| 372 |
+
]
|
| 373 |
+
)
|
| 374 |
lines.extend(
|
| 375 |
[
|
| 376 |
"",
|