anhtld commited on
Commit
de20dc2
·
verified ·
1 Parent(s): 922c22c

Auto-sync: 2026-06-28 20:44:54 (part 3)

Browse files
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4s040_safe_margin0p20_mean_by_type_noop0p025_wg0p02_summary.json ADDED
@@ -0,0 +1,322 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
3
+ "objective": "near_miss_policy_bc5",
4
+ "out_name": "policy_rollout_retrieval_residual_k4s040_safe_margin0p20_mean_by_type_noop0p025_wg0p02.json",
5
+ "num_completed": 3,
6
+ "baseline_h4_policy_success": 0.2967,
7
+ "baseline_h16_rank_checkpoint_success": 0.29739130434782607,
8
+ "mean_success": 0.35246376811594204,
9
+ "std_success": 0.014164396200429698,
10
+ "mean_progress": 0.5669270546512079,
11
+ "mean_action_mse_to_best": 0.39555692845307616,
12
+ "gain_vs_h4": 0.055763768115942025,
13
+ "gain_vs_h16_rank_checkpoint": 0.055072463768115976,
14
+ "rows": [
15
+ {
16
+ "seed": 0,
17
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_k4s040_safe_margin0p20_mean_by_type_noop0p025_wg0p02.json",
18
+ "num_groups": 575,
19
+ "selection_mode": "retrieval_residual",
20
+ "num_candidates": 6,
21
+ "candidate_sigma": 0.0,
22
+ "selection_margin": 0.2,
23
+ "prepend_policy_candidate": false,
24
+ "field_optim_steps": 0,
25
+ "field_optim_step_size": 0.0,
26
+ "field_optim_trust_radius": 0.0,
27
+ "field_optim_l2_penalty": 0.0,
28
+ "retrieval_neighbors": 4,
29
+ "retrieval_metric": "raw",
30
+ "retrieval_type_min_success": 0.0,
31
+ "retrieval_residual_scale": 0.4,
32
+ "retrieval_residual_scales": [],
33
+ "retrieval_residual_anchor": "expert",
34
+ "retrieval_residual_reduce": "mean_by_type",
35
+ "candidate_type_bonuses": {
36
+ "residual_no_op": 0.025,
37
+ "residual_wrong_gripper": 0.02
38
+ },
39
+ "selected_residual_scale_counts": {
40
+ "0.4": 575
41
+ },
42
+ "policy_rollout_success_rate": 0.34608695652173915,
43
+ "policy_rollout_progress": 0.551369690887792,
44
+ "oracle_success_rate": 0.8573913043478261,
45
+ "action_mse_to_best": 0.38197267810611624,
46
+ "best_policy_val": {
47
+ "bc_loss": 0.13721593966086706,
48
+ "field_effect_loss": 0.009290305380192068,
49
+ "field_potential_loss": 0.2666468388504452,
50
+ "field_preference_loss": 0.5130573478009965,
51
+ "lattice_edges": 3833.3333333333335,
52
+ "progress_mae": 0.1933159919248687,
53
+ "rank_acc": 0.8265031774838766,
54
+ "rank_loss": 0.5130523675017886,
55
+ "regret_mae": 0.3756548762321472,
56
+ "success_accuracy": 0.8773836526605818,
57
+ "total_loss": 1.5581054819954767
58
+ },
59
+ "per_task": {
60
+ "LiftPegUpright-v1": {
61
+ "action_mse_to_best": 0.31969772460726426,
62
+ "expert_success_rate": 0.8865979381443299,
63
+ "num_groups": 97,
64
+ "oracle_success_rate": 0.9278350515463918,
65
+ "policy_expert_regret": 1.0783334703175063,
66
+ "policy_oracle_regret": 1.1047489480259491,
67
+ "policy_rollout_progress": 0.5714634232914325,
68
+ "policy_rollout_success_rate": 0.2268041237113402,
69
+ "restore_max_error": 2.384185791015625e-07
70
+ },
71
+ "PickCube-v1": {
72
+ "action_mse_to_best": 0.3026410622450595,
73
+ "expert_success_rate": 0.9375,
74
+ "num_groups": 208,
75
+ "oracle_success_rate": 0.9471153846153846,
76
+ "policy_expert_regret": 0.961157998499962,
77
+ "policy_oracle_regret": 0.9745808659264674,
78
+ "policy_rollout_progress": 0.6161086453268161,
79
+ "policy_rollout_success_rate": 0.34134615384615385,
80
+ "restore_max_error": 2.384185791015625e-07
81
+ },
82
+ "PullCube-v1": {
83
+ "action_mse_to_best": 0.6028722151146306,
84
+ "expert_success_rate": 0.19480519480519481,
85
+ "num_groups": 77,
86
+ "oracle_success_rate": 0.36363636363636365,
87
+ "policy_expert_regret": 0.24377496006547825,
88
+ "policy_oracle_regret": 0.41733099815931607,
89
+ "policy_rollout_progress": 0.312481374268166,
90
+ "policy_rollout_success_rate": 0.2077922077922078,
91
+ "restore_max_error": 2.3096799850463867e-07
92
+ },
93
+ "PushCube-v1": {
94
+ "action_mse_to_best": 0.36465501579198434,
95
+ "expert_success_rate": 0.8617021276595744,
96
+ "num_groups": 94,
97
+ "oracle_success_rate": 0.9893617021276596,
98
+ "policy_expert_regret": 0.5508309629686335,
99
+ "policy_oracle_regret": 0.647163599729538,
100
+ "policy_rollout_progress": 0.6932619321853557,
101
+ "policy_rollout_success_rate": 0.6595744680851063,
102
+ "restore_max_error": 2.384185791015625e-07
103
+ },
104
+ "StackCube-v1": {
105
+ "action_mse_to_best": 0.45429835988754275,
106
+ "expert_success_rate": 0.6767676767676768,
107
+ "num_groups": 99,
108
+ "oracle_success_rate": 0.8585858585858586,
109
+ "policy_expert_regret": 0.9403071221378114,
110
+ "policy_oracle_regret": 1.10289698026397,
111
+ "policy_rollout_progress": 0.4467407528198127,
112
+ "policy_rollout_success_rate": 0.2828282828282828,
113
+ "restore_max_error": 2.384185791015625e-07
114
+ }
115
+ }
116
+ },
117
+ {
118
+ "seed": 1,
119
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_k4s040_safe_margin0p20_mean_by_type_noop0p025_wg0p02.json",
120
+ "num_groups": 575,
121
+ "selection_mode": "retrieval_residual",
122
+ "num_candidates": 6,
123
+ "candidate_sigma": 0.0,
124
+ "selection_margin": 0.2,
125
+ "prepend_policy_candidate": false,
126
+ "field_optim_steps": 0,
127
+ "field_optim_step_size": 0.0,
128
+ "field_optim_trust_radius": 0.0,
129
+ "field_optim_l2_penalty": 0.0,
130
+ "retrieval_neighbors": 4,
131
+ "retrieval_metric": "raw",
132
+ "retrieval_type_min_success": 0.0,
133
+ "retrieval_residual_scale": 0.4,
134
+ "retrieval_residual_scales": [],
135
+ "retrieval_residual_anchor": "expert",
136
+ "retrieval_residual_reduce": "mean_by_type",
137
+ "candidate_type_bonuses": {
138
+ "residual_no_op": 0.025,
139
+ "residual_wrong_gripper": 0.02
140
+ },
141
+ "selected_residual_scale_counts": {
142
+ "0.4": 575
143
+ },
144
+ "policy_rollout_success_rate": 0.3426086956521739,
145
+ "policy_rollout_progress": 0.5665413653461829,
146
+ "oracle_success_rate": 0.8695652173913043,
147
+ "action_mse_to_best": 0.3883226982811871,
148
+ "best_policy_val": {
149
+ "bc_loss": 0.1315989200439718,
150
+ "field_effect_loss": 0.010110370814800262,
151
+ "field_potential_loss": 0.31546536915832096,
152
+ "field_preference_loss": 0.4960637440284093,
153
+ "lattice_edges": 3833.3333333333335,
154
+ "progress_mae": 0.19023226698239645,
155
+ "rank_acc": 0.8255469501018524,
156
+ "rank_loss": 0.4960531195004781,
157
+ "regret_mae": 0.42156532737943864,
158
+ "success_accuracy": 0.8807473679383596,
159
+ "total_loss": 1.561984618504842
160
+ },
161
+ "per_task": {
162
+ "LiftPegUpright-v1": {
163
+ "action_mse_to_best": 0.3255710795406351,
164
+ "expert_success_rate": 0.8584070796460177,
165
+ "num_groups": 113,
166
+ "oracle_success_rate": 0.9380530973451328,
167
+ "policy_expert_regret": 0.9873926745051831,
168
+ "policy_oracle_regret": 1.0665947045900126,
169
+ "policy_rollout_progress": 0.6355170211960784,
170
+ "policy_rollout_success_rate": 0.23008849557522124,
171
+ "restore_max_error": 2.384185791015625e-07
172
+ },
173
+ "PickCube-v1": {
174
+ "action_mse_to_best": 0.3200199794550629,
175
+ "expert_success_rate": 0.9402173913043478,
176
+ "num_groups": 184,
177
+ "oracle_success_rate": 0.9456521739130435,
178
+ "policy_expert_regret": 1.0707762707429735,
179
+ "policy_oracle_regret": 1.0708002752179038,
180
+ "policy_rollout_progress": 0.5744255784818012,
181
+ "policy_rollout_success_rate": 0.29347826086956524,
182
+ "restore_max_error": 2.384185791015625e-07
183
+ },
184
+ "PullCube-v1": {
185
+ "action_mse_to_best": 0.5715771112591028,
186
+ "expert_success_rate": 0.25,
187
+ "num_groups": 76,
188
+ "oracle_success_rate": 0.40789473684210525,
189
+ "policy_expert_regret": 0.30013844498286124,
190
+ "policy_oracle_regret": 0.5234715515061429,
191
+ "policy_rollout_progress": 0.28913191000097677,
192
+ "policy_rollout_success_rate": 0.18421052631578946,
193
+ "restore_max_error": 2.384185791015625e-07
194
+ },
195
+ "PushCube-v1": {
196
+ "action_mse_to_best": 0.35765871029716356,
197
+ "expert_success_rate": 0.8198198198198198,
198
+ "num_groups": 111,
199
+ "oracle_success_rate": 1.0,
200
+ "policy_expert_regret": 0.32971009934270706,
201
+ "policy_oracle_regret": 0.38007361851296984,
202
+ "policy_rollout_progress": 0.8181245796852283,
203
+ "policy_rollout_success_rate": 0.8018018018018018,
204
+ "restore_max_error": 2.3795291781425476e-07
205
+ },
206
+ "StackCube-v1": {
207
+ "action_mse_to_best": 0.48870731873826667,
208
+ "expert_success_rate": 0.6923076923076923,
209
+ "num_groups": 91,
210
+ "oracle_success_rate": 0.8571428571428571,
211
+ "policy_expert_regret": 1.1411885159028756,
212
+ "policy_oracle_regret": 1.3008503630593582,
213
+ "policy_rollout_progress": 0.38975496410013555,
214
+ "policy_rollout_success_rate": 0.15384615384615385,
215
+ "restore_max_error": 1.9371509552001953e-07
216
+ }
217
+ }
218
+ },
219
+ {
220
+ "seed": 2,
221
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_k4s040_safe_margin0p20_mean_by_type_noop0p025_wg0p02.json",
222
+ "num_groups": 575,
223
+ "selection_mode": "retrieval_residual",
224
+ "num_candidates": 6,
225
+ "candidate_sigma": 0.0,
226
+ "selection_margin": 0.2,
227
+ "prepend_policy_candidate": false,
228
+ "field_optim_steps": 0,
229
+ "field_optim_step_size": 0.0,
230
+ "field_optim_trust_radius": 0.0,
231
+ "field_optim_l2_penalty": 0.0,
232
+ "retrieval_neighbors": 4,
233
+ "retrieval_metric": "raw",
234
+ "retrieval_type_min_success": 0.0,
235
+ "retrieval_residual_scale": 0.4,
236
+ "retrieval_residual_scales": [],
237
+ "retrieval_residual_anchor": "expert",
238
+ "retrieval_residual_reduce": "mean_by_type",
239
+ "candidate_type_bonuses": {
240
+ "residual_no_op": 0.025,
241
+ "residual_wrong_gripper": 0.02
242
+ },
243
+ "selected_residual_scale_counts": {
244
+ "0.4": 575
245
+ },
246
+ "policy_rollout_success_rate": 0.36869565217391304,
247
+ "policy_rollout_progress": 0.5828701077196488,
248
+ "oracle_success_rate": 0.8765217391304347,
249
+ "action_mse_to_best": 0.41637540897192515,
250
+ "best_policy_val": {
251
+ "bc_loss": 0.11367896075050037,
252
+ "field_effect_loss": 0.009670218582161598,
253
+ "field_potential_loss": 0.2641640139950646,
254
+ "field_preference_loss": 0.5130490180518892,
255
+ "lattice_edges": 3833.3333333333335,
256
+ "progress_mae": 0.2021729110015763,
257
+ "rank_acc": 0.8333857821093665,
258
+ "rank_loss": 0.5130119257503085,
259
+ "regret_mae": 0.3958987047274907,
260
+ "success_accuracy": 0.8680730561415354,
261
+ "total_loss": 1.4394984311527677
262
+ },
263
+ "per_task": {
264
+ "LiftPegUpright-v1": {
265
+ "action_mse_to_best": 0.35425721921395353,
266
+ "expert_success_rate": 0.8229166666666666,
267
+ "num_groups": 96,
268
+ "oracle_success_rate": 0.9270833333333334,
269
+ "policy_expert_regret": 0.8476849501021206,
270
+ "policy_oracle_regret": 0.9539546399998168,
271
+ "policy_rollout_progress": 0.6463205499264101,
272
+ "policy_rollout_success_rate": 0.3333333333333333,
273
+ "restore_max_error": 1.955777406692505e-07
274
+ },
275
+ "PickCube-v1": {
276
+ "action_mse_to_best": 0.335480088330429,
277
+ "expert_success_rate": 0.9444444444444444,
278
+ "num_groups": 198,
279
+ "oracle_success_rate": 0.9595959595959596,
280
+ "policy_expert_regret": 0.9807514474924767,
281
+ "policy_oracle_regret": 0.9878348980211851,
282
+ "policy_rollout_progress": 0.6357372662409989,
283
+ "policy_rollout_success_rate": 0.3383838383838384,
284
+ "restore_max_error": 2.384185791015625e-07
285
+ },
286
+ "PullCube-v1": {
287
+ "action_mse_to_best": 0.6275676680935753,
288
+ "expert_success_rate": 0.24444444444444444,
289
+ "num_groups": 90,
290
+ "oracle_success_rate": 0.4666666666666667,
291
+ "policy_expert_regret": 0.3140720418619821,
292
+ "policy_oracle_regret": 0.5427349449100802,
293
+ "policy_rollout_progress": 0.3188257521984269,
294
+ "policy_rollout_success_rate": 0.2111111111111111,
295
+ "restore_max_error": 2.384185791015625e-07
296
+ },
297
+ "PushCube-v1": {
298
+ "action_mse_to_best": 0.3851195577495169,
299
+ "expert_success_rate": 0.8514851485148515,
300
+ "num_groups": 101,
301
+ "oracle_success_rate": 1.0,
302
+ "policy_expert_regret": 0.3620445999768701,
303
+ "policy_oracle_regret": 0.41694685138098087,
304
+ "policy_rollout_progress": 0.8008749307972369,
305
+ "policy_rollout_success_rate": 0.7821782178217822,
306
+ "restore_max_error": 2.384185791015625e-07
307
+ },
308
+ "StackCube-v1": {
309
+ "action_mse_to_best": 0.4844882684863276,
310
+ "expert_success_rate": 0.7666666666666667,
311
+ "num_groups": 90,
312
+ "oracle_success_rate": 0.9111111111111111,
313
+ "policy_expert_regret": 1.1607355892658233,
314
+ "policy_oracle_regret": 1.2970718792743152,
315
+ "policy_rollout_progress": 0.4182763857973946,
316
+ "policy_rollout_success_rate": 0.16666666666666666,
317
+ "restore_max_error": 2.2351741790771484e-07
318
+ }
319
+ }
320
+ }
321
+ ]
322
+ }
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4s040_safe_margin0p20_mean_by_type_noop0p025_wg0p02_summary.md ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # h=16 Best-Policy Checkpoint Rollout
2
+
3
+ Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
4
+ Objective: `near_miss_policy_bc5`
5
+ Result file: `policy_rollout_retrieval_residual_k4s040_safe_margin0p20_mean_by_type_noop0p025_wg0p02.json`
6
+ Completed seeds: 3
7
+ Baseline h=4 policy success: 29.67%
8
+ Baseline h=16 rank-checkpoint success: 29.74%
9
+
10
+ Mean success: 35.25% +/- 1.42%
11
+ Gain vs h=16 rank checkpoint: +5.51%
12
+ Mean progress: 56.69%
13
+ Mean action MSE to best: 0.396
14
+
15
+ | seed | mode | k | policy cand | retrieval K | retrieval metric | residual anchor | residual reduce | min type success | residual scale | residual scales | margin | sigma | opt steps | trust | success | progress | oracle | action MSE |
16
+ |---:|---|---:|---|---:|---|---|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|
17
+ | 0 | retrieval_residual | 6 | no | 4 | raw | expert | mean_by_type | 0.00 | 0.40 | none | 0.200 | 0.00 | 0 | 0.00 | 34.61% | 55.14% | 85.74% | 0.382 |
18
+ | 1 | retrieval_residual | 6 | no | 4 | raw | expert | mean_by_type | 0.00 | 0.40 | none | 0.200 | 0.00 | 0 | 0.00 | 34.26% | 56.65% | 86.96% | 0.388 |
19
+ | 2 | retrieval_residual | 6 | no | 4 | raw | expert | mean_by_type | 0.00 | 0.40 | none | 0.200 | 0.00 | 0 | 0.00 | 36.87% | 58.29% | 87.65% | 0.416 |
results/paper_analysis.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "best_clean_key": "residual_k4_consensus_noopbonus003",
3
- "generated_utc": "2026-06-29T00:39:30+00:00",
4
  "mechanism_gap": {
5
  "best_clean_vs_direct_same_ckpt": 0.06956521739130428,
6
  "best_clean_vs_h16": 0.05507246376811592,
 
1
  {
2
  "best_clean_key": "residual_k4_consensus_noopbonus003",
3
+ "generated_utc": "2026-06-29T00:41:07+00:00",
4
  "mechanism_gap": {
5
  "best_clean_vs_direct_same_ckpt": 0.06956521739130428,
6
  "best_clean_vs_h16": 0.05507246376811592,
results/paper_analysis.md CHANGED
@@ -1,6 +1,6 @@
1
  # Paper Analysis
2
 
3
- Generated: `2026-06-29T00:39:30+00:00`
4
 
5
  ## Main Seed Statistics
6
 
 
1
  # Paper Analysis
2
 
3
+ Generated: `2026-06-29T00:41:07+00:00`
4
 
5
  ## Main Seed Statistics
6
 
results/paper_core_results.md CHANGED
@@ -40,6 +40,7 @@ and the remaining clean-to-same-state proposal gap is `+21.74 pp`.
40
  | K2 train-state residual retrieval, safe residuals + advantage margin 0.20 | No | No | 35.01% | +5.28 pp | Previous best clean diagnostic; abstention makes a small train-neighborhood useful |
41
  | K4 train-state residual retrieval, safe residuals + mean-by-type tangent consensus | No | No | 34.96% | +5.22 pp | Near-tie clean diagnostic; consensus alone does not beat raw K2 residuals |
42
  | K4 mean-by-type residual retrieval + no-op prior 0.03 | No | No | 35.25% | +5.51 pp | Current best clean diagnostic; 0.025-0.035 forms a small plateau that nudges high-value no-op residuals without changing the core proposal family |
 
43
  | K1 train-state residual ray-search, tight scales | No | No | 34.84% | +5.10 pp | Scale-grid ray-search is a near-tie but does not beat the typed-prior clean row |
44
  | K2 train-state residual ray-search, tight scales | No | No | 34.84% | +5.10 pp | More scale choices along the same local rays do not improve the clean row |
45
  | K2 train-state residual ray-search, broad scales | No | No | 34.96% | +5.22 pp | Best ray-search row, still below the typed-prior clean row |
@@ -77,14 +78,15 @@ Suggested main-table rows:
77
  12. K2 train-state residual retrieval, typed safe families + advantage margin 0.20
78
  13. K4 train-state residual retrieval, mean-by-type tangent consensus
79
  14. K4 mean-by-type residual retrieval + no-op prior plateau, canonical 0.03
80
- 15. K2 broad tangent ray-search
81
- 16. Residual-tangent distillation policy
82
- 17. Residual+Gaussian hybrid, K32 sigma0.35
83
- 18. Lattice, near-miss only
84
- 19. Lattice, no expert
85
- 20. Lattice, no expert + policy baseline candidate
86
- 21. Lattice, full
87
- 22. Oracle ceiling
 
88
 
89
  Suggested claim:
90
 
@@ -93,7 +95,7 @@ Suggested claim:
93
  > abstention and a small typed no-op prior plateau gives the strongest clean gain so far, while ungated KNN residual
94
  > retrieval, field-gradient ascent, broader non-expert BC targets, field-teacher/tangent distillation, z-score retrieval,
95
  > train-family reliability priors, policy-relative anchoring, residual+Gaussian hybrids,
96
- > tangent consensus, tangent ray-search, and same-state policy-baseline fallback fail to improve the main rows.
97
  > The large effect appears only when the field is queried on
98
  > same-state intervention proposals, and the mechanism is isolated to local near-miss
99
  > counterfactual geometry.
 
40
  | K2 train-state residual retrieval, safe residuals + advantage margin 0.20 | No | No | 35.01% | +5.28 pp | Previous best clean diagnostic; abstention makes a small train-neighborhood useful |
41
  | K4 train-state residual retrieval, safe residuals + mean-by-type tangent consensus | No | No | 34.96% | +5.22 pp | Near-tie clean diagnostic; consensus alone does not beat raw K2 residuals |
42
  | K4 mean-by-type residual retrieval + no-op prior 0.03 | No | No | 35.25% | +5.51 pp | Current best clean diagnostic; 0.025-0.035 forms a small plateau that nudges high-value no-op residuals without changing the core proposal family |
43
+ | K4 mean-by-type residual retrieval + wrong-gripper typed prior | No | No | 35.19-35.25% | +5.45-5.51 pp | Wrong-gripper-only is lower and two-family priors only tie the no-op plateau; useful negative/tie diagnostic |
44
  | K1 train-state residual ray-search, tight scales | No | No | 34.84% | +5.10 pp | Scale-grid ray-search is a near-tie but does not beat the typed-prior clean row |
45
  | K2 train-state residual ray-search, tight scales | No | No | 34.84% | +5.10 pp | More scale choices along the same local rays do not improve the clean row |
46
  | K2 train-state residual ray-search, broad scales | No | No | 34.96% | +5.22 pp | Best ray-search row, still below the typed-prior clean row |
 
78
  12. K2 train-state residual retrieval, typed safe families + advantage margin 0.20
79
  13. K4 train-state residual retrieval, mean-by-type tangent consensus
80
  14. K4 mean-by-type residual retrieval + no-op prior plateau, canonical 0.03
81
+ 15. K4 mean-by-type residual retrieval + wrong-gripper typed-prior diagnostics
82
+ 16. K2 broad tangent ray-search
83
+ 17. Residual-tangent distillation policy
84
+ 18. Residual+Gaussian hybrid, K32 sigma0.35
85
+ 19. Lattice, near-miss only
86
+ 20. Lattice, no expert
87
+ 21. Lattice, no expert + policy baseline candidate
88
+ 22. Lattice, full
89
+ 23. Oracle ceiling
90
 
91
  Suggested claim:
92
 
 
95
  > abstention and a small typed no-op prior plateau gives the strongest clean gain so far, while ungated KNN residual
96
  > retrieval, field-gradient ascent, broader non-expert BC targets, field-teacher/tangent distillation, z-score retrieval,
97
  > train-family reliability priors, policy-relative anchoring, residual+Gaussian hybrids,
98
+ > tangent consensus, tangent ray-search, wrong-gripper typed priors, and same-state policy-baseline fallback fail to improve the main rows.
99
  > The large effect appears only when the field is queried on
100
  > same-state intervention proposals, and the mechanism is isolated to local near-miss
101
  > counterfactual geometry.
results/paper_story_memo.md CHANGED
@@ -30,6 +30,7 @@ when queried on proposal geometry that matches those local counterfactuals.
30
  | Tangent consensus is close but needs sparse typing | K4 mean-by-type residual consensus reaches 34.96%; a small no-op residual prior plateau at 0.025-0.035 raises it to 35.25% | Current best clean result |
31
  | Tangent ray-search does not beat the typed-prior clean row | K1/K2 tight scale-grid ray search reach 34.84%; K2 broad reaches 34.96%; K4 tight reaches 34.55%, all below the no-op-prior row at 35.25% | Near-tie/negative diagnostic |
32
  | Typed no-op residual prior improves the clean bridge | CPU smoke `14883591` passed; bonuses 0.025/0.03/0.035 tie at 35.25%, while 0.01/0.02/0.05/0.08 are slightly lower | Current best clean diagnostic |
 
33
  | The proposal gap is now quantified | `paper_analysis.md` reports best clean +5.51 pp over canonical h16, same-state no-expert +27.25 pp, leaving a +21.74 pp clean-to-same-state gap | Core paper tension |
34
  | Policy fallback is not the same-state mechanism | adding a policy baseline candidate to the no-expert same-state lattice drops 56.99% to 40.70% even with margin 0.00 | Negative diagnostic |
35
  | Z-score retrieval metric does not help | z-score rows reach 32.23-32.81%, below raw retrieval | Negative diagnostic |
@@ -58,18 +59,19 @@ clean proposal result, the intended main rows are:
58
  13. K2 train-state residual retrieval, typed safe families + advantage margin: 35.01%
59
  14. K4 mean-by-type tangent consensus: 34.96%
60
  15. K4 mean-by-type tangent consensus + typed no-op prior 0.025-0.035: 35.25%
61
- 16. K2 broad tangent ray-search: 34.96%
62
- 17. K1/K2 tight tangent ray-search: 34.84% / 34.84%
63
- 18. K4 tight tangent ray-search: 34.55%
64
- 19. Residual-tangent distillation policy: 28.87%
65
- 20. Z-score residual retrieval: 32.23-32.81%
66
- 21. Train-family reliability prior: 33.28-33.33%
67
- 22. Residual+Gaussian hybrid K32/K64: 31.30% / 30.90%
68
- 23. Lattice, near-miss only: 55.94%
69
- 24. Lattice, no expert: 56.99%
70
- 25. Lattice, no expert + policy baseline candidate: 40.70%
71
- 26. Lattice, full: 69.33%
72
- 27. Oracle ceiling: 86.78%
 
73
 
74
  ## Novelty Framing
75
 
@@ -97,7 +99,7 @@ test-time search. The cleaner novelty is:
97
 
98
  ## Job Status
99
 
100
- Last checked: `2026-06-28 22:10 UTC`. The counterfactual tangent ray-search batch
101
  completed, and the typed no-op residual-prior clean sweep completed after a passing
102
  CPU smoke.
103
 
@@ -166,6 +168,15 @@ CPU smoke.
166
  35.19%, 35.25%, and 35.25%; `0.025`/`0.03`/`0.035` form a small best plateau.
167
  Summary jobs `14884376`/`14884378`/`14884380`/`14884382` and rebuild job
168
  `14884383` completed.
 
 
 
 
 
 
 
 
 
169
  - `14869627`: completed CPU Apptainer smoke for the new residual scale-grid
170
  selector. It selected index `3` on a two-residual/two-scale toy case and
171
  returned the expected action `0.20`, validating the candidate expansion and
@@ -194,5 +205,6 @@ CPU smoke.
194
  selection histograms when writing reviewer-facing tables.
195
  - Treat z-score retrieval, repaired train-family reliability priors, Gaussian hybrids,
196
  field optimization, field-teacher/tangent distillation, policy-relative anchoring, tangent consensus,
 
197
  and same-state policy-baseline fallback as negative or near-tie diagnostics
198
  that sharpen the story around local counterfactual proposal geometry.
 
30
  | Tangent consensus is close but needs sparse typing | K4 mean-by-type residual consensus reaches 34.96%; a small no-op residual prior plateau at 0.025-0.035 raises it to 35.25% | Current best clean result |
31
  | Tangent ray-search does not beat the typed-prior clean row | K1/K2 tight scale-grid ray search reach 34.84%; K2 broad reaches 34.96%; K4 tight reaches 34.55%, all below the no-op-prior row at 35.25% | Near-tie/negative diagnostic |
32
  | Typed no-op residual prior improves the clean bridge | CPU smoke `14883591` passed; bonuses 0.025/0.03/0.035 tie at 35.25%, while 0.01/0.02/0.05/0.08 are slightly lower | Current best clean diagnostic |
33
+ | Wrong-gripper typed prior does not add a new clean bridge | wrong-gripper-only reaches 35.19%; no-op+wrong-gripper 0.02 ties 35.25%; no-op+wrong-gripper 0.04 drops to 35.13% | Negative/tie diagnostic |
34
  | The proposal gap is now quantified | `paper_analysis.md` reports best clean +5.51 pp over canonical h16, same-state no-expert +27.25 pp, leaving a +21.74 pp clean-to-same-state gap | Core paper tension |
35
  | Policy fallback is not the same-state mechanism | adding a policy baseline candidate to the no-expert same-state lattice drops 56.99% to 40.70% even with margin 0.00 | Negative diagnostic |
36
  | Z-score retrieval metric does not help | z-score rows reach 32.23-32.81%, below raw retrieval | Negative diagnostic |
 
59
  13. K2 train-state residual retrieval, typed safe families + advantage margin: 35.01%
60
  14. K4 mean-by-type tangent consensus: 34.96%
61
  15. K4 mean-by-type tangent consensus + typed no-op prior 0.025-0.035: 35.25%
62
+ 16. Wrong-gripper prior / no-op+wrong-gripper prior: 35.19% / 35.25%
63
+ 17. K2 broad tangent ray-search: 34.96%
64
+ 18. K1/K2 tight tangent ray-search: 34.84% / 34.84%
65
+ 19. K4 tight tangent ray-search: 34.55%
66
+ 20. Residual-tangent distillation policy: 28.87%
67
+ 21. Z-score residual retrieval: 32.23-32.81%
68
+ 22. Train-family reliability prior: 33.28-33.33%
69
+ 23. Residual+Gaussian hybrid K32/K64: 31.30% / 30.90%
70
+ 24. Lattice, near-miss only: 55.94%
71
+ 25. Lattice, no expert: 56.99%
72
+ 26. Lattice, no expert + policy baseline candidate: 40.70%
73
+ 27. Lattice, full: 69.33%
74
+ 28. Oracle ceiling: 86.78%
75
 
76
  ## Novelty Framing
77
 
 
99
 
100
  ## Job Status
101
 
102
+ Last checked: `2026-06-29 00:41 UTC`. The counterfactual tangent ray-search batch
103
  completed, and the typed no-op residual-prior clean sweep completed after a passing
104
  CPU smoke.
105
 
 
168
  35.19%, 35.25%, and 35.25%; `0.025`/`0.03`/`0.035` form a small best plateau.
169
  Summary jobs `14884376`/`14884378`/`14884380`/`14884382` and rebuild job
170
  `14884383` completed.
171
+ - `14890019`: completed CPU smoke for two-family candidate-type bonuses
172
+ (`residual_no_op=0.03`, `residual_wrong_gripper=0.02`), validating multi-type
173
+ parsing before GPU rollout.
174
+ - `14890071`/`14890073`/`14890075`/`14890077`: completed wrong-gripper and
175
+ no-op+wrong-gripper typed-prior GPU sweeps. Results are 35.19% for
176
+ wrong-gripper-only, 35.25% for no-op 0.03 + wrong-gripper 0.02, 35.13% for
177
+ no-op 0.03 + wrong-gripper 0.04, and 35.25% for no-op 0.025 + wrong-gripper
178
+ 0.02. Summary jobs `14890072`/`14890074`/`14890076`/`14890078` and rebuild
179
+ job `14890079` completed.
180
  - `14869627`: completed CPU Apptainer smoke for the new residual scale-grid
181
  selector. It selected index `3` on a two-residual/two-scale toy case and
182
  returned the expected action `0.20`, validating the candidate expansion and
 
205
  selection histograms when writing reviewer-facing tables.
206
  - Treat z-score retrieval, repaired train-family reliability priors, Gaussian hybrids,
207
  field optimization, field-teacher/tangent distillation, policy-relative anchoring, tangent consensus,
208
+ wrong-gripper typed priors,
209
  and same-state policy-baseline fallback as negative or near-tie diagnostics
210
  that sharpen the story around local counterfactual proposal geometry.
scripts/build_paper_analysis.py CHANGED
@@ -59,6 +59,38 @@ METHODS = [
59
  "k4s040_safe_margin0p20_mean_by_type_summary.json"
60
  ),
61
  ),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62
  MethodSpec(
63
  key="residual_k4_consensus_noopbonus003",
64
  label="K4 mean-by-type tangent consensus, no-op bonus 0.03",
@@ -521,6 +553,10 @@ def _render_markdown(report: dict[str, Any]) -> str:
521
  for key in [
522
  "best_clean_residual_k2",
523
  "residual_k4_consensus",
 
 
 
 
524
  "residual_k4_consensus_noopbonus003",
525
  "residual_k4_consensus_noopbonus001",
526
  "residual_k4_consensus_noopbonus002",
 
59
  "k4s040_safe_margin0p20_mean_by_type_summary.json"
60
  ),
61
  ),
62
+ MethodSpec(
63
+ key="residual_k4_kernel_consensus",
64
+ label="K4 kernel-weighted tangent consensus",
65
+ summary_path=(
66
+ "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
67
+ "k4s040_safe_margin0p20_kernel_mean_by_type_summary.json"
68
+ ),
69
+ ),
70
+ MethodSpec(
71
+ key="residual_k4_kernel_consensus_noopbonus003",
72
+ label="K4 kernel-weighted tangent consensus, no-op bonus 0.03",
73
+ summary_path=(
74
+ "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
75
+ "k4s040_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json"
76
+ ),
77
+ ),
78
+ MethodSpec(
79
+ key="residual_k4_kernel_consensus_s035_noopbonus003",
80
+ label="K4 kernel-weighted tangent consensus, scale 0.35, no-op bonus 0.03",
81
+ summary_path=(
82
+ "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
83
+ "k4s035_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json"
84
+ ),
85
+ ),
86
+ MethodSpec(
87
+ key="residual_k4_kernel_consensus_s045_noopbonus003",
88
+ label="K4 kernel-weighted tangent consensus, scale 0.45, no-op bonus 0.03",
89
+ summary_path=(
90
+ "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
91
+ "k4s045_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json"
92
+ ),
93
+ ),
94
  MethodSpec(
95
  key="residual_k4_consensus_noopbonus003",
96
  label="K4 mean-by-type tangent consensus, no-op bonus 0.03",
 
553
  for key in [
554
  "best_clean_residual_k2",
555
  "residual_k4_consensus",
556
+ "residual_k4_kernel_consensus",
557
+ "residual_k4_kernel_consensus_noopbonus003",
558
+ "residual_k4_kernel_consensus_s035_noopbonus003",
559
+ "residual_k4_kernel_consensus_s045_noopbonus003",
560
  "residual_k4_consensus_noopbonus003",
561
  "residual_k4_consensus_noopbonus001",
562
  "residual_k4_consensus_noopbonus002",
scripts/build_paper_table_status.py CHANGED
@@ -335,6 +335,46 @@ SPECS = [
335
  story_role="counterfactual tangent consensus near-tie ablation",
336
  pending_job="14868699/14868700",
337
  ),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
338
  ResultSpec(
339
  key="retrieval_residual_k4_mean_noopbonus003",
340
  label="K4 mean-by-type residual retrieval, scale 0.40, margin 0.20, no-op residual bonus 0.03",
 
335
  story_role="counterfactual tangent consensus near-tie ablation",
336
  pending_job="14868699/14868700",
337
  ),
338
+ ResultSpec(
339
+ key="retrieval_residual_k4_kernel_mean",
340
+ label="K4 kernel-weighted residual retrieval, scale 0.40, margin 0.20",
341
+ path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4s040_safe_margin0p20_kernel_mean_by_type_summary.json",
342
+ clean_deployment="yes",
343
+ same_state_proposals="no",
344
+ expert_proposal="no",
345
+ story_role="local counterfactual tangent-field interpolation",
346
+ pending_job="14891067/14891083",
347
+ ),
348
+ ResultSpec(
349
+ key="retrieval_residual_k4_kernel_mean_noopbonus003",
350
+ label="K4 kernel-weighted residual retrieval, scale 0.40, margin 0.20, no-op residual bonus 0.03",
351
+ path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4s040_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json",
352
+ clean_deployment="yes",
353
+ same_state_proposals="no",
354
+ expert_proposal="no",
355
+ story_role="local counterfactual tangent-field interpolation with sparse-action prior",
356
+ pending_job="14891072/14891085",
357
+ ),
358
+ ResultSpec(
359
+ key="retrieval_residual_k4_kernel_mean_s035_noopbonus003",
360
+ label="K4 kernel-weighted residual retrieval, scale 0.35, margin 0.20, no-op residual bonus 0.03",
361
+ path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4s035_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json",
362
+ clean_deployment="yes",
363
+ same_state_proposals="no",
364
+ expert_proposal="no",
365
+ story_role="local counterfactual tangent-field interpolation scale check",
366
+ pending_job="14891076/14891087",
367
+ ),
368
+ ResultSpec(
369
+ key="retrieval_residual_k4_kernel_mean_s045_noopbonus003",
370
+ label="K4 kernel-weighted residual retrieval, scale 0.45, margin 0.20, no-op residual bonus 0.03",
371
+ path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4s045_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json",
372
+ clean_deployment="yes",
373
+ same_state_proposals="no",
374
+ expert_proposal="no",
375
+ story_role="local counterfactual tangent-field interpolation scale check",
376
+ pending_job="14891082/14891088",
377
+ ),
378
  ResultSpec(
379
  key="retrieval_residual_k4_mean_noopbonus003",
380
  label="K4 mean-by-type residual retrieval, scale 0.40, margin 0.20, no-op residual bonus 0.03",
scripts/eval_maniskill_policy_rollout.py CHANGED
@@ -148,9 +148,10 @@ def main(argv: list[str] | None = None) -> int:
148
  )
149
  parser.add_argument(
150
  "--retrieval-residual-reduce",
151
- choices=("none", "mean_by_type", "median_by_type"),
152
  default="none",
153
- help="Optional consensus reduction over retrieved residuals with the same candidate type.",
 
154
  )
155
  parser.add_argument(
156
  "--lattice-exclude-types",
 
148
  )
149
  parser.add_argument(
150
  "--retrieval-residual-reduce",
151
+ choices=("none", "mean_by_type", "median_by_type", "kernel_mean_by_type"),
152
  default="none",
153
+ help="Optional consensus reduction over retrieved residuals with the same candidate type. "
154
+ "'kernel_mean_by_type' weights source residuals by train-state retrieval distance.",
155
  )
156
  parser.add_argument(
157
  "--lattice-exclude-types",