anhtld commited on
Commit
dfb4bfa
·
verified ·
1 Parent(s): a861f94

Auto-sync: 2026-06-30 02:54:08

Browse files
results/paper_analysis.json CHANGED
@@ -1,11 +1,11 @@
1
  {
2
- "best_clean_key": "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003",
3
- "generated_utc": "2026-06-30T06:41:21+00:00",
4
  "mechanism_gap": {
5
- "best_clean_vs_direct_same_ckpt": 0.07304347826086954,
6
- "best_clean_vs_h16": 0.058550724637681184,
7
  "same_state_full_vs_no_expert": 0.12347826086956515,
8
- "same_state_no_expert_vs_best_clean": 0.21391304347826096,
9
  "same_state_no_expert_vs_h16": 0.27246376811594214
10
  },
11
  "methods": {
@@ -1433,9 +1433,116 @@
1433
  "std_success": 0.009889114266221456
1434
  },
1435
  "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8": {
 
 
 
 
 
 
 
 
 
 
 
1436
  "label": "K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 diagnostic",
1437
- "missing": true,
1438
- "source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_oraclek8_summary.json"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1439
  },
1440
  "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_srcscorebonus002": {
1441
  "ci95_success": 0.02131295542962397,
@@ -7097,38 +7204,38 @@
7097
  },
7098
  "paired_deltas": {
7099
  "best_clean - canonical_h16": {
7100
- "ci95_delta": 0.04370704391798567,
7101
- "left": "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003",
7102
- "mean_delta": 0.05855072463768116,
7103
  "right": "h16_policy_canonical",
7104
  "seed_deltas": {
7105
- "0": 0.06956521739130433,
7106
  "1": 0.038260869565217404,
7107
- "2": 0.06782608695652176
7108
  },
7109
  "seeds": [
7110
  0,
7111
  1,
7112
  2
7113
  ],
7114
- "std_delta": 0.01759303293390552
7115
  },
7116
  "best_clean - direct_same_ckpt": {
7117
- "ci95_delta": 0.04121578446082111,
7118
- "left": "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003",
7119
- "mean_delta": 0.07304347826086956,
7120
  "right": "near_miss_policy_bc5",
7121
  "seed_deltas": {
7122
- "0": 0.07478260869565218,
7123
  "1": 0.055652173913043446,
7124
- "2": 0.08869565217391306
7125
  },
7126
  "seeds": [
7127
  0,
7128
  1,
7129
  2
7130
  ],
7131
- "std_delta": 0.0165902469811643
7132
  },
7133
  "full_lattice - no_expert_lattice": {
7134
  "ci95_delta": 0.02628111194117426,
@@ -7186,16 +7293,16 @@
7186
  "best_clean_vs_h16": {
7187
  "LiftPegUpright-v1": 0.03278131303915899,
7188
  "PickCube-v1": 0.1219595621769535,
7189
- "PullCube-v1": 0.006887167413483192,
7190
- "PushCube-v1": 0.006846429323783432,
7191
- "StackCube-v1": 0.0594738594738595
7192
  },
7193
  "no_expert_vs_best_clean": {
7194
  "LiftPegUpright-v1": 0.35374851747103375,
7195
  "PickCube-v1": 0.29366195398804096,
7196
- "PullCube-v1": -0.006261866788182552,
7197
- "PushCube-v1": 0.043639384064915965,
7198
- "StackCube-v1": 0.282077182077182
7199
  }
7200
  }
7201
  }
 
1
  {
2
+ "best_clean_key": "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8",
3
+ "generated_utc": "2026-06-30T06:57:02+00:00",
4
  "mechanism_gap": {
5
+ "best_clean_vs_direct_same_ckpt": 0.07362318840579707,
6
+ "best_clean_vs_h16": 0.05913043478260871,
7
  "same_state_full_vs_no_expert": 0.12347826086956515,
8
+ "same_state_no_expert_vs_best_clean": 0.21333333333333343,
9
  "same_state_no_expert_vs_h16": 0.27246376811594214
10
  },
11
  "methods": {
 
1433
  "std_success": 0.009889114266221456
1434
  },
1435
  "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8": {
1436
+ "candidate_oracle_rollouts": 8,
1437
+ "candidate_oracle_type_counts": {
1438
+ "retrieval_residual_policy_residual": 867,
1439
+ "retrieval_residual_residual_near_miss": 201,
1440
+ "retrieval_residual_residual_near_miss+residual_wrong_gripper": 99,
1441
+ "retrieval_residual_residual_no_op": 182,
1442
+ "retrieval_residual_residual_no_op+residual_wrong_gripper": 86,
1443
+ "retrieval_residual_residual_wrong_gripper": 290
1444
+ },
1445
+ "candidate_oracle_unique_tolerance": 1e-06,
1446
+ "ci95_success": 0.03024411731871433,
1447
  "label": "K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 diagnostic",
1448
+ "mean_action_mse_to_best": 0.4063519326314006,
1449
+ "mean_candidate_oracle_improvement_rate": 0.4602898550724637,
1450
+ "mean_candidate_oracle_progress": 0.6443418554713328,
1451
+ "mean_candidate_oracle_score_gain_over_selected": 0.16034898876064066,
1452
+ "mean_candidate_oracle_success_rate": 0.4307246376811594,
1453
+ "mean_candidate_oracle_unique_count": NaN,
1454
+ "mean_progress": 0.571575072284851,
1455
+ "mean_success": 0.3565217391304348,
1456
+ "num_completed": 3,
1457
+ "per_task_success": {
1458
+ "LiftPegUpright-v1": {
1459
+ "mean_num_groups": 102.0,
1460
+ "mean_success": 0.26340865087329823,
1461
+ "std_success": 0.06057881385012102
1462
+ },
1463
+ "PickCube-v1": {
1464
+ "mean_num_groups": 196.66666666666666,
1465
+ "mean_success": 0.3278169093386485,
1466
+ "std_success": 0.02533117351097172
1467
+ },
1468
+ "PullCube-v1": {
1469
+ "mean_num_groups": 81.0,
1470
+ "mean_success": 0.21716032505506191,
1471
+ "std_success": 0.0182535211270701
1472
+ },
1473
+ "PushCube-v1": {
1474
+ "mean_num_groups": 102.0,
1475
+ "mean_success": 0.7541548289389031,
1476
+ "std_success": 0.08244271787813129
1477
+ },
1478
+ "StackCube-v1": {
1479
+ "mean_num_groups": 93.33333333333333,
1480
+ "mean_success": 0.19707329707329704,
1481
+ "std_success": 0.0831467196651622
1482
+ }
1483
+ },
1484
+ "seed_action_mse_to_best": {
1485
+ "0": 0.3930192190424904,
1486
+ "1": 0.39824521353549286,
1487
+ "2": 0.42779136531621864
1488
+ },
1489
+ "seed_progress": {
1490
+ "0": 0.5579317773906919,
1491
+ "1": 0.571580135051323,
1492
+ "2": 0.5852133044125383
1493
+ },
1494
+ "seed_success": {
1495
+ "0": 0.35130434782608694,
1496
+ "1": 0.34782608695652173,
1497
+ "2": 0.37043478260869567
1498
+ },
1499
+ "selected_candidate_type_counts": {
1500
+ "retrieval_residual_policy_residual": 1613,
1501
+ "retrieval_residual_residual_near_miss": 8,
1502
+ "retrieval_residual_residual_near_miss+residual_wrong_gripper": 5,
1503
+ "retrieval_residual_residual_no_op": 56,
1504
+ "retrieval_residual_residual_no_op+residual_wrong_gripper": 24,
1505
+ "retrieval_residual_residual_wrong_gripper": 19
1506
+ },
1507
+ "selected_residual_scale_counts": {
1508
+ "0.35": 1631,
1509
+ "0.4": 11,
1510
+ "0.45": 83
1511
+ },
1512
+ "selected_type_outcomes": {
1513
+ "retrieval_residual_policy_residual": {
1514
+ "count": 1613.0,
1515
+ "mean_progress": 0.561925975703087,
1516
+ "success_rate": 0.34593924364538126
1517
+ },
1518
+ "retrieval_residual_residual_near_miss": {
1519
+ "count": 8.0,
1520
+ "mean_progress": 0.8754628524184227,
1521
+ "success_rate": 0.75
1522
+ },
1523
+ "retrieval_residual_residual_near_miss+residual_wrong_gripper": {
1524
+ "count": 5.0,
1525
+ "mean_progress": 0.7557869911193847,
1526
+ "success_rate": 0.6
1527
+ },
1528
+ "retrieval_residual_residual_no_op": {
1529
+ "count": 56.0,
1530
+ "mean_progress": 0.7249722566921264,
1531
+ "success_rate": 0.5
1532
+ },
1533
+ "retrieval_residual_residual_no_op+residual_wrong_gripper": {
1534
+ "count": 24.0,
1535
+ "mean_progress": 0.6711457151298722,
1536
+ "success_rate": 0.4583333333333333
1537
+ },
1538
+ "retrieval_residual_residual_wrong_gripper": {
1539
+ "count": 19.0,
1540
+ "mean_progress": 0.6364115562878156,
1541
+ "success_rate": 0.47368421052631576
1542
+ }
1543
+ },
1544
+ "source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_oraclek8_summary.json",
1545
+ "std_success": 0.012173913043478276
1546
  },
1547
  "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_srcscorebonus002": {
1548
  "ci95_success": 0.02131295542962397,
 
7204
  },
7205
  "paired_deltas": {
7206
  "best_clean - canonical_h16": {
7207
+ "ci95_delta": 0.04510826498554671,
7208
+ "left": "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8",
7209
+ "mean_delta": 0.05913043478260869,
7210
  "right": "h16_policy_canonical",
7211
  "seed_deltas": {
7212
+ "0": 0.0678260869565217,
7213
  "1": 0.038260869565217404,
7214
+ "2": 0.07130434782608697
7215
  },
7216
  "seeds": [
7217
  0,
7218
  1,
7219
  2
7220
  ],
7221
+ "std_delta": 0.018157054798105293
7222
  },
7223
  "best_clean - direct_same_ckpt": {
7224
+ "ci95_delta": 0.045383317930687055,
7225
+ "left": "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8",
7226
+ "mean_delta": 0.07362318840579708,
7227
  "right": "near_miss_policy_bc5",
7228
  "seed_deltas": {
7229
+ "0": 0.07304347826086954,
7230
  "1": 0.055652173913043446,
7231
+ "2": 0.09217391304347827
7232
  },
7233
  "seeds": [
7234
  0,
7235
  1,
7236
  2
7237
  ],
7238
+ "std_delta": 0.018267769572856522
7239
  },
7240
  "full_lattice - no_expert_lattice": {
7241
  "ci95_delta": 0.02628111194117426,
 
7293
  "best_clean_vs_h16": {
7294
  "LiftPegUpright-v1": 0.03278131303915899,
7295
  "PickCube-v1": 0.1219595621769535,
7296
+ "PullCube-v1": 0.019305840358471965,
7297
+ "PushCube-v1": 0.003597657063003523,
7298
+ "StackCube-v1": 0.05610685610685609
7299
  },
7300
  "no_expert_vs_best_clean": {
7301
  "LiftPegUpright-v1": 0.35374851747103375,
7302
  "PickCube-v1": 0.29366195398804096,
7303
+ "PullCube-v1": -0.018680539733171325,
7304
+ "PushCube-v1": 0.04688815632569587,
7305
+ "StackCube-v1": 0.2854441854441855
7306
  }
7307
  }
7308
  }
results/paper_analysis.md CHANGED
@@ -1,6 +1,6 @@
1
  # Paper Analysis
2
 
3
- Generated: `2026-06-30T06:41:21+00:00`
4
 
5
  ## Main Seed Statistics
6
 
@@ -50,7 +50,7 @@ Generated: `2026-06-30T06:41:21+00:00`
50
  | residual_k4_composemasked_dropnmnoop_grid035040045 | K4 composed type-consensus tangents, masked, drop near-miss+no-op composite | 3 | 35.48% +/- 1.25 | +/- 3.12 | 57.00% | 0.406 | +5.74 pp |
51
  | residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003 | K4 composed type-consensus tangents, masked, drop near-miss+no-op composite | 3 | 35.59% +/- 0.99 | +/- 2.46 | 57.07% | 0.406 | +5.86 pp |
52
  | residual_k4_composemasked_dropnmnoop_grid035040045_margin010_noopbonus003 | K4 compatible tangents, margin 0.10, no-op bonus 0.03 | 3 | 34.67% +/- 0.96 | +/- 2.38 | 56.45% | 0.418 | +4.93 pp |
53
- | residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8 | K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 diagnostic | 0 | missing | missing | missing | missing | missing |
54
  | residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001 | K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.01 | 3 | 35.59% +/- 0.99 | +/- 2.46 | 57.10% | 0.406 | +5.86 pp |
55
  | residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus002 | K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.02 | 3 | 35.59% +/- 0.99 | +/- 2.46 | 57.10% | 0.406 | +5.86 pp |
56
  | residual_k4_composemasked_dropnmnoop_grid035040045_srcscorebonus002 | K4 composed type-consensus tangents, masked, drop near-miss+no-op composite, source-score bonus 0.02 | 3 | 35.48% +/- 1.22 | +/- 3.02 | 57.02% | 0.408 | +5.74 pp |
@@ -98,8 +98,8 @@ Generated: `2026-06-30T06:41:21+00:00`
98
 
99
  | comparison | seeds | mean delta | 95% CI | seed deltas |
100
  |---|---:|---:|---:|---|
101
- | best_clean - canonical_h16 | 3 | +5.86 pp | +/- 4.37 | 0:+6.96, 1:+3.83, 2:+6.78 |
102
- | best_clean - direct_same_ckpt | 3 | +7.30 pp | +/- 4.12 | 0:+7.48, 1:+5.57, 2:+8.87 |
103
  | no_expert_lattice - canonical_h16 | 3 | +27.25 pp | +/- 8.58 | 0:+23.30, 1:+28.70, 2:+29.74 |
104
  | full_lattice - no_expert_lattice | 3 | +12.35 pp | +/- 2.63 | 0:+13.57, 1:+11.83, 2:+11.65 |
105
  | policy_candidate_lattice - no_expert_lattice | 3 | -16.29 pp | +/- 7.55 | 0:-15.48, 1:-13.74, 2:-19.65 |
@@ -110,25 +110,32 @@ Generated: `2026-06-30T06:41:21+00:00`
110
  |---|---:|---:|---:|---:|---:|---:|---:|
111
  | LiftPegUpright-v1 | 23.06% | 26.34% | 62.90% | 61.72% | 73.19% | +3.28 pp | +35.37 pp |
112
  | PickCube-v1 | 20.59% | 32.78% | 58.13% | 62.15% | 84.19% | +12.20 pp | +29.37 pp |
113
- | PullCube-v1 | 19.79% | 20.47% | 15.02% | 19.85% | 22.41% | +0.69 pp | -0.63 pp |
114
- | PushCube-v1 | 75.06% | 75.74% | 82.54% | 80.10% | 81.92% | +0.68 pp | +4.36 pp |
115
- | StackCube-v1 | 14.10% | 20.04% | 50.41% | 48.25% | 60.83% | +5.95 pp | +28.21 pp |
116
 
117
  ## Mechanism Gap
118
 
119
- - Best clean residual transport improves over canonical h16 by +5.86 pp.
120
  - Same-state no-expert lattice improves over canonical h16 by +27.25 pp.
121
- - Remaining clean-to-same-state proposal gap is +21.39 pp.
122
  - Full lattice adds expert proposals and reaches 69.33%, a +12.35 pp gain over no-expert.
123
 
 
 
 
 
 
 
 
124
  ## Selection Histograms
125
 
126
  - `same_state_near_miss`: lattice_near_miss=1725 (100.0%)
127
  - `same_state_no_expert`: lattice_near_miss=1263 (73.2%), lattice_no_op=222 (12.9%), lattice_random_negative=144 (8.3%), lattice_wrong_gripper=62 (3.6%), lattice_wrong_direction=34 (2.0%)
128
  - `same_state_policy_baseline`: policy_continuous=1022 (59.2%), lattice_near_miss=448 (26.0%), lattice_no_op=119 (6.9%), lattice_random_negative=75 (4.3%), lattice_wrong_gripper=45 (2.6%), lattice_wrong_direction=16 (0.9%)
129
  - `same_state_full`: lattice_expert=977 (56.6%), lattice_near_miss=348 (20.2%), lattice_no_op=177 (10.3%), lattice_random_negative=138 (8.0%), lattice_wrong_gripper=55 (3.2%), lattice_wrong_direction=30 (1.7%)
130
- - `residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003`: retrieval_residual_policy_residual=1613 (93.5%), retrieval_residual_residual_no_op=56 (3.2%), retrieval_residual_residual_no_op+residual_wrong_gripper=24 (1.4%), retrieval_residual_residual_wrong_gripper=19 (1.1%), retrieval_residual_residual_near_miss=8 (0.5%), retrieval_residual_residual_near_miss+residual_wrong_gripper=5 (0.3%)
131
- - `residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003` residual scale counts: {'0.35': 1631, '0.4': 11, '0.45': 83}
132
 
133
  ## Selected-Type Outcomes
134
 
 
1
  # Paper Analysis
2
 
3
+ Generated: `2026-06-30T06:57:02+00:00`
4
 
5
  ## Main Seed Statistics
6
 
 
50
  | residual_k4_composemasked_dropnmnoop_grid035040045 | K4 composed type-consensus tangents, masked, drop near-miss+no-op composite | 3 | 35.48% +/- 1.25 | +/- 3.12 | 57.00% | 0.406 | +5.74 pp |
51
  | residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003 | K4 composed type-consensus tangents, masked, drop near-miss+no-op composite | 3 | 35.59% +/- 0.99 | +/- 2.46 | 57.07% | 0.406 | +5.86 pp |
52
  | residual_k4_composemasked_dropnmnoop_grid035040045_margin010_noopbonus003 | K4 compatible tangents, margin 0.10, no-op bonus 0.03 | 3 | 34.67% +/- 0.96 | +/- 2.38 | 56.45% | 0.418 | +4.93 pp |
53
+ | residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8 | K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 diagnostic | 3 | 35.65% +/- 1.22 | +/- 3.02 | 57.16% | 0.406 | +5.91 pp |
54
  | residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001 | K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.01 | 3 | 35.59% +/- 0.99 | +/- 2.46 | 57.10% | 0.406 | +5.86 pp |
55
  | residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus002 | K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.02 | 3 | 35.59% +/- 0.99 | +/- 2.46 | 57.10% | 0.406 | +5.86 pp |
56
  | residual_k4_composemasked_dropnmnoop_grid035040045_srcscorebonus002 | K4 composed type-consensus tangents, masked, drop near-miss+no-op composite, source-score bonus 0.02 | 3 | 35.48% +/- 1.22 | +/- 3.02 | 57.02% | 0.408 | +5.74 pp |
 
98
 
99
  | comparison | seeds | mean delta | 95% CI | seed deltas |
100
  |---|---:|---:|---:|---|
101
+ | best_clean - canonical_h16 | 3 | +5.91 pp | +/- 4.51 | 0:+6.78, 1:+3.83, 2:+7.13 |
102
+ | best_clean - direct_same_ckpt | 3 | +7.36 pp | +/- 4.54 | 0:+7.30, 1:+5.57, 2:+9.22 |
103
  | no_expert_lattice - canonical_h16 | 3 | +27.25 pp | +/- 8.58 | 0:+23.30, 1:+28.70, 2:+29.74 |
104
  | full_lattice - no_expert_lattice | 3 | +12.35 pp | +/- 2.63 | 0:+13.57, 1:+11.83, 2:+11.65 |
105
  | policy_candidate_lattice - no_expert_lattice | 3 | -16.29 pp | +/- 7.55 | 0:-15.48, 1:-13.74, 2:-19.65 |
 
110
  |---|---:|---:|---:|---:|---:|---:|---:|
111
  | LiftPegUpright-v1 | 23.06% | 26.34% | 62.90% | 61.72% | 73.19% | +3.28 pp | +35.37 pp |
112
  | PickCube-v1 | 20.59% | 32.78% | 58.13% | 62.15% | 84.19% | +12.20 pp | +29.37 pp |
113
+ | PullCube-v1 | 19.79% | 21.72% | 15.02% | 19.85% | 22.41% | +1.93 pp | -1.87 pp |
114
+ | PushCube-v1 | 75.06% | 75.42% | 82.54% | 80.10% | 81.92% | +0.36 pp | +4.69 pp |
115
+ | StackCube-v1 | 14.10% | 19.71% | 50.41% | 48.25% | 60.83% | +5.61 pp | +28.54 pp |
116
 
117
  ## Mechanism Gap
118
 
119
+ - Best clean residual transport improves over canonical h16 by +5.91 pp.
120
  - Same-state no-expert lattice improves over canonical h16 by +27.25 pp.
121
+ - Remaining clean-to-same-state proposal gap is +21.33 pp.
122
  - Full lattice adds expert proposals and reaches 69.33%, a +12.35 pp gain over no-expert.
123
 
124
+ ## Candidate-Oracle Diagnostic
125
+
126
+ - Oracle over the deployed candidate prefix reaches 43.07% with mean progress 64.43%; this is diagnostic-only because it uses measured rollout outcomes after generating candidates.
127
+ - Mean oracle-prefix score gain over the selected branch is +0.160, which isolates ranking/abstention headroom inside the clean proposal set.
128
+ - Mean unique candidates in the prefix: nan.
129
+ - Candidate-oracle best type counts: {'retrieval_residual_policy_residual': 867, 'retrieval_residual_residual_near_miss': 201, 'retrieval_residual_residual_near_miss+residual_wrong_gripper': 99, 'retrieval_residual_residual_no_op': 182, 'retrieval_residual_residual_no_op+residual_wrong_gripper': 86, 'retrieval_residual_residual_wrong_gripper': 290}.
130
+
131
  ## Selection Histograms
132
 
133
  - `same_state_near_miss`: lattice_near_miss=1725 (100.0%)
134
  - `same_state_no_expert`: lattice_near_miss=1263 (73.2%), lattice_no_op=222 (12.9%), lattice_random_negative=144 (8.3%), lattice_wrong_gripper=62 (3.6%), lattice_wrong_direction=34 (2.0%)
135
  - `same_state_policy_baseline`: policy_continuous=1022 (59.2%), lattice_near_miss=448 (26.0%), lattice_no_op=119 (6.9%), lattice_random_negative=75 (4.3%), lattice_wrong_gripper=45 (2.6%), lattice_wrong_direction=16 (0.9%)
136
  - `same_state_full`: lattice_expert=977 (56.6%), lattice_near_miss=348 (20.2%), lattice_no_op=177 (10.3%), lattice_random_negative=138 (8.0%), lattice_wrong_gripper=55 (3.2%), lattice_wrong_direction=30 (1.7%)
137
+ - `residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8`: retrieval_residual_policy_residual=1613 (93.5%), retrieval_residual_residual_no_op=56 (3.2%), retrieval_residual_residual_no_op+residual_wrong_gripper=24 (1.4%), retrieval_residual_residual_wrong_gripper=19 (1.1%), retrieval_residual_residual_near_miss=8 (0.5%), retrieval_residual_residual_near_miss+residual_wrong_gripper=5 (0.3%)
138
+ - `residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8` residual scale counts: {'0.35': 1631, '0.4': 11, '0.45': 83}
139
 
140
  ## Selected-Type Outcomes
141
 
results/paper_table_status.json CHANGED
@@ -1363,14 +1363,14 @@
1363
  "story_role": "diagnostic oracle over generated clean candidate prefix to separate proposal headroom from field ranking",
1364
  "fallback_success": null,
1365
  "pending_job": "14953513/14953522",
1366
- "path_exists": false,
1367
- "status": "pending",
1368
- "success": null,
1369
- "std_success": null,
1370
  "completed_seeds": null,
1371
- "num_completed": null,
1372
  "best_config": null,
1373
- "gain_vs_h16_policy": null
1374
  },
1375
  {
1376
  "key": "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001",
@@ -2618,6 +2618,7 @@
2618
  "KNN counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.",
2619
  "K4 train-state residual retrieval, scale 0.40, safe residuals, mean-by-type tangent consensus improves the clean bridge but is not yet the main result.",
2620
  "Policy-baseline abstention inside the same-state no-expert lattice drops from 56.99% to 40.70%, so the mechanism result should emphasize counterfactual proposal geometry rather than policy fallback.",
2621
- "Singleton near-miss priors 0.01/0.02 tie the clean best rather than improving it; the compatible chart preserves high-precision near-miss tangents, but they remain sparse."
 
2622
  ]
2623
  }
 
1363
  "story_role": "diagnostic oracle over generated clean candidate prefix to separate proposal headroom from field ranking",
1364
  "fallback_success": null,
1365
  "pending_job": "14953513/14953522",
1366
+ "path_exists": true,
1367
+ "status": "complete",
1368
+ "success": 0.43072463768115943,
1369
+ "std_success": 0.012173913043478276,
1370
  "completed_seeds": null,
1371
+ "num_completed": 3,
1372
  "best_config": null,
1373
+ "gain_vs_h16_policy": 0.13333333333333336
1374
  },
1375
  {
1376
  "key": "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001",
 
2618
  "KNN counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.",
2619
  "K4 train-state residual retrieval, scale 0.40, safe residuals, mean-by-type tangent consensus improves the clean bridge but is not yet the main result.",
2620
  "Policy-baseline abstention inside the same-state no-expert lattice drops from 56.99% to 40.70%, so the mechanism result should emphasize counterfactual proposal geometry rather than policy fallback.",
2621
+ "Singleton near-miss priors 0.01/0.02 tie the clean best rather than improving it; the compatible chart preserves high-precision near-miss tangents, but they remain sparse.",
2622
+ "Candidate-oracle prefix K=8 is diagnostic-only; compare it to the clean selected row to decide whether the next paper move should be proposal generation or field ranking."
2623
  ]
2624
  }
results/paper_table_status.md CHANGED
@@ -74,7 +74,7 @@ Baseline h=16 policy: 29.74%
74
  | retrieval_residual_k4_composemasked_dropnmnoop_grid035040045 | K4 composed type-consensus residual retrieval, masked, drop near-miss+no-op composite | complete | 35.48% | +5.74 pp | yes | no | no | exact compatibility mask without typed no-op prior |
75
  | retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003 | K4 composed type-consensus residual retrieval, masked, drop near-miss+no-op composite | complete | 35.59% | +5.86 pp | yes | no | no | exact incompatibility mask for one weak composed tangent pair |
76
  | retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_margin010_noopbonus003 | K4 compatible residual retrieval, margin 0.10, no-op bonus 0.03 | complete | 34.67% | +4.93 pp | yes | no | no | oracle-motivated abstention sensitivity on the compatible local tangent chart |
77
- | retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8 | K4 compatible residual retrieval, unique candidate-oracle prefix K=8 diagnostic | pending 14953513/14953522 | pending | pending | diagnostic | no | no | diagnostic oracle over generated clean candidate prefix to separate proposal headroom from field ranking |
78
  | retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001 | K4 composed compatible residual retrieval, no-op bonus 0.03, singleton near-miss bonus 0.01 | complete | 35.59% | +5.86 pp | yes | no | no | revive high-precision singleton near-miss tangents without boosting toxic near-miss+no-op composites |
79
  | retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus002 | K4 composed compatible residual retrieval, no-op bonus 0.03, singleton near-miss bonus 0.02 | complete | 35.59% | +5.86 pp | yes | no | no | stronger singleton near-miss revival prior on the compatible local tangent chart |
80
  | retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_srcscorebonus002 | K4 composed type-consensus residual retrieval, masked, drop near-miss+no-op composite, source-score bonus 0.02 | complete | 35.48% | +5.74 pp | yes | no | no | train-measured source-score prior on the compatible local tangent chart |
@@ -151,3 +151,4 @@ Baseline h=16 policy: 29.74%
151
  - K4 train-state residual retrieval, scale 0.40, safe residuals, mean-by-type tangent consensus improves the clean bridge but is not yet the main result.
152
  - Policy-baseline abstention inside the same-state no-expert lattice drops from 56.99% to 40.70%, so the mechanism result should emphasize counterfactual proposal geometry rather than policy fallback.
153
  - Singleton near-miss priors 0.01/0.02 tie the clean best rather than improving it; the compatible chart preserves high-precision near-miss tangents, but they remain sparse.
 
 
74
  | retrieval_residual_k4_composemasked_dropnmnoop_grid035040045 | K4 composed type-consensus residual retrieval, masked, drop near-miss+no-op composite | complete | 35.48% | +5.74 pp | yes | no | no | exact compatibility mask without typed no-op prior |
75
  | retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003 | K4 composed type-consensus residual retrieval, masked, drop near-miss+no-op composite | complete | 35.59% | +5.86 pp | yes | no | no | exact incompatibility mask for one weak composed tangent pair |
76
  | retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_margin010_noopbonus003 | K4 compatible residual retrieval, margin 0.10, no-op bonus 0.03 | complete | 34.67% | +4.93 pp | yes | no | no | oracle-motivated abstention sensitivity on the compatible local tangent chart |
77
+ | retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8 | K4 compatible residual retrieval, unique candidate-oracle prefix K=8 diagnostic | complete | 43.07% | +13.33 pp | diagnostic | no | no | diagnostic oracle over generated clean candidate prefix to separate proposal headroom from field ranking |
78
  | retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001 | K4 composed compatible residual retrieval, no-op bonus 0.03, singleton near-miss bonus 0.01 | complete | 35.59% | +5.86 pp | yes | no | no | revive high-precision singleton near-miss tangents without boosting toxic near-miss+no-op composites |
79
  | retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus002 | K4 composed compatible residual retrieval, no-op bonus 0.03, singleton near-miss bonus 0.02 | complete | 35.59% | +5.86 pp | yes | no | no | stronger singleton near-miss revival prior on the compatible local tangent chart |
80
  | retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_srcscorebonus002 | K4 composed type-consensus residual retrieval, masked, drop near-miss+no-op composite, source-score bonus 0.02 | complete | 35.48% | +5.74 pp | yes | no | no | train-measured source-score prior on the compatible local tangent chart |
 
151
  - K4 train-state residual retrieval, scale 0.40, safe residuals, mean-by-type tangent consensus improves the clean bridge but is not yet the main result.
152
  - Policy-baseline abstention inside the same-state no-expert lattice drops from 56.99% to 40.70%, so the mechanism result should emphasize counterfactual proposal geometry rather than policy fallback.
153
  - Singleton near-miss priors 0.01/0.02 tie the clean best rather than improving it; the compatible chart preserves high-precision near-miss tangents, but they remain sparse.
154
+ - Candidate-oracle prefix K=8 is diagnostic-only; compare it to the clean selected row to decide whether the next paper move should be proposal generation or field ranking.
scripts/slurm/summarize_h16_policy_ckpt.sbatch CHANGED
@@ -132,6 +132,21 @@ for result_path in sorted(base_dir.glob(f"seed_*/{out_name}")):
132
  "candidate_oracle_type_counts": data.get(
133
  "candidate_oracle_type_counts", {}
134
  ),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
135
  "selected_residual_scale_counts": dict(selected_scale_counts),
136
  "policy_rollout_success_rate": data.get("policy_rollout_success_rate", 0.0),
137
  "policy_rollout_progress": data.get("policy_rollout_progress", 0.0),
@@ -168,6 +183,21 @@ summary = {
168
  "rows": rows,
169
  }
170
  if candidate_oracle_rows:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
171
  summary.update(
172
  {
173
  "candidate_oracle_rollouts": max(
@@ -225,6 +255,34 @@ if candidate_oracle_rows:
225
  Counter(),
226
  )
227
  ),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
228
  }
229
  )
230
 
@@ -286,6 +344,33 @@ if candidate_oracle_rows:
286
  ),
287
  ]
288
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
289
  lines.extend(
290
  [
291
  "",
 
132
  "candidate_oracle_type_counts": data.get(
133
  "candidate_oracle_type_counts", {}
134
  ),
135
+ "candidate_oracle_best_branch_rank": data.get(
136
+ "candidate_oracle_best_branch_rank"
137
+ ),
138
+ "candidate_oracle_best_branch_rank_counts": data.get(
139
+ "candidate_oracle_best_branch_rank_counts", {}
140
+ ),
141
+ "candidate_oracle_branch_success_rates": data.get(
142
+ "candidate_oracle_branch_success_rates", []
143
+ ),
144
+ "candidate_oracle_branch_progress": data.get(
145
+ "candidate_oracle_branch_progress", []
146
+ ),
147
+ "candidate_oracle_branch_score_gains_over_selected": data.get(
148
+ "candidate_oracle_branch_score_gains_over_selected", []
149
+ ),
150
  "selected_residual_scale_counts": dict(selected_scale_counts),
151
  "policy_rollout_success_rate": data.get("policy_rollout_success_rate", 0.0),
152
  "policy_rollout_progress": data.get("policy_rollout_progress", 0.0),
 
183
  "rows": rows,
184
  }
185
  if candidate_oracle_rows:
186
+ def mean_branch_metric(key: str) -> list[float]:
187
+ max_len = max(
188
+ (len(row.get(key) or []) for row in candidate_oracle_rows),
189
+ default=0,
190
+ )
191
+ values = []
192
+ for branch_index in range(max_len):
193
+ branch_values = [
194
+ row[key][branch_index]
195
+ for row in candidate_oracle_rows
196
+ if len(row.get(key) or []) > branch_index
197
+ ]
198
+ values.append(statistics.mean(branch_values) if branch_values else 0.0)
199
+ return values
200
+
201
  summary.update(
202
  {
203
  "candidate_oracle_rollouts": max(
 
255
  Counter(),
256
  )
257
  ),
258
+ "mean_candidate_oracle_best_branch_rank": statistics.mean(
259
+ row["candidate_oracle_best_branch_rank"]
260
+ for row in candidate_oracle_rows
261
+ if row.get("candidate_oracle_best_branch_rank") is not None
262
+ )
263
+ if any(
264
+ row.get("candidate_oracle_best_branch_rank") is not None
265
+ for row in candidate_oracle_rows
266
+ )
267
+ else None,
268
+ "candidate_oracle_best_branch_rank_counts": dict(
269
+ sum(
270
+ (
271
+ Counter(row.get("candidate_oracle_best_branch_rank_counts", {}))
272
+ for row in candidate_oracle_rows
273
+ ),
274
+ Counter(),
275
+ )
276
+ ),
277
+ "mean_candidate_oracle_branch_success_rates": mean_branch_metric(
278
+ "candidate_oracle_branch_success_rates"
279
+ ),
280
+ "mean_candidate_oracle_branch_progress": mean_branch_metric(
281
+ "candidate_oracle_branch_progress"
282
+ ),
283
+ "mean_candidate_oracle_branch_score_gains_over_selected": (
284
+ mean_branch_metric("candidate_oracle_branch_score_gains_over_selected")
285
+ ),
286
  }
287
  )
288
 
 
344
  ),
345
  ]
346
  )
347
+ if summary.get("mean_candidate_oracle_best_branch_rank") is not None:
348
+ branch_success = ", ".join(
349
+ f"{value:.2%}"
350
+ for value in summary.get(
351
+ "mean_candidate_oracle_branch_success_rates", []
352
+ )
353
+ )
354
+ branch_gains = ", ".join(
355
+ f"{value:+.3f}"
356
+ for value in summary.get(
357
+ "mean_candidate_oracle_branch_score_gains_over_selected", []
358
+ )
359
+ )
360
+ lines.extend(
361
+ [
362
+ (
363
+ "Candidate-oracle best branch rank: "
364
+ f"{summary['mean_candidate_oracle_best_branch_rank']:.2f}"
365
+ ),
366
+ (
367
+ "Candidate-oracle best branch rank counts: "
368
+ f"{summary['candidate_oracle_best_branch_rank_counts']}"
369
+ ),
370
+ f"Candidate-oracle branch success by rank: {branch_success}",
371
+ f"Candidate-oracle branch gain by rank: {branch_gains}",
372
+ ]
373
+ )
374
  lines.extend(
375
  [
376
  "",