Hugging Face's logo

anhtld
/
vla

Model card Files
xet
 Community
anhtld commited on
Commit
ea98068
·
verified ·
1 Parent(s): 4038c72

Auto-sync: 2026-06-28 02:19:01

Browse files
Files changed (10) hide show
  1. results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p25_type_success0p25_summary.json +286 -0
  2. results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p25_type_success0p25_summary.md +19 -0
  3. results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p50_type_success0p10_summary.json +286 -0
  4. results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p50_type_success0p10_summary.md +19 -0
  5. results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p50_type_success0p25_summary.json +286 -0
  6. results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p50_type_success0p25_summary.md +19 -0
  7. results/paper_story_memo.md +6 -0
  8. results/paper_table_status.json +38 -0
  9. results/paper_table_status.md +2 -0
  10. scripts/build_paper_table_status.py +20 -0
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p25_type_success0p25_summary.json ADDED
@@ -0,0 +1,286 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
3
+ "objective": "near_miss_policy_bc5",
4
+ "out_name": "policy_rollout_retrieval_residual_scale0p25_type_success0p25.json",
5
+ "num_completed": 3,
6
+ "baseline_h4_policy_success": 0.2967,
7
+ "baseline_h16_rank_checkpoint_success": 0.29739130434782607,
8
+ "mean_success": 0.3292753623188406,
9
+ "std_success": 0.015194599909659314,
10
+ "mean_progress": 0.5523768832874687,
11
+ "mean_action_mse_to_best": 0.40910649155253087,
12
+ "gain_vs_h4": 0.03257536231884056,
13
+ "gain_vs_h16_rank_checkpoint": 0.03188405797101451,
14
+ "rows": [
15
+ {
16
+ "seed": 0,
17
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_scale0p25_type_success0p25.json",
18
+ "num_groups": 575,
19
+ "selection_mode": "retrieval_residual",
20
+ "num_candidates": 16,
21
+ "candidate_sigma": 0.0,
22
+ "field_optim_steps": 0,
23
+ "field_optim_step_size": 0.0,
24
+ "field_optim_trust_radius": 0.0,
25
+ "field_optim_l2_penalty": 0.0,
26
+ "retrieval_neighbors": 1,
27
+ "retrieval_metric": "raw",
28
+ "retrieval_type_min_success": 0.25,
29
+ "retrieval_residual_scale": 0.25,
30
+ "policy_rollout_success_rate": 0.32521739130434785,
31
+ "policy_rollout_progress": 0.5427134724674018,
32
+ "oracle_success_rate": 0.8573913043478261,
33
+ "action_mse_to_best": 0.39587786793789786,
34
+ "best_policy_val": {
35
+ "bc_loss": 0.13721593966086706,
36
+ "field_effect_loss": 0.009290305380192068,
37
+ "field_potential_loss": 0.2666468388504452,
38
+ "field_preference_loss": 0.5130573478009965,
39
+ "lattice_edges": 3833.3333333333335,
40
+ "progress_mae": 0.1933159919248687,
41
+ "rank_acc": 0.8265031774838766,
42
+ "rank_loss": 0.5130523675017886,
43
+ "regret_mae": 0.3756548762321472,
44
+ "success_accuracy": 0.8773836526605818,
45
+ "total_loss": 1.5581054819954767
46
+ },
47
+ "per_task": {
48
+ "LiftPegUpright-v1": {
49
+ "action_mse_to_best": 0.32431400695470036,
50
+ "expert_success_rate": 0.8865979381443299,
51
+ "num_groups": 97,
52
+ "oracle_success_rate": 0.9278350515463918,
53
+ "policy_expert_regret": 1.0923672899142982,
54
+ "policy_oracle_regret": 1.1187827676227413,
55
+ "policy_rollout_progress": 0.567738882045156,
56
+ "policy_rollout_success_rate": 0.21649484536082475,
57
+ "restore_max_error": 4.76837158203125e-07
58
+ },
59
+ "PickCube-v1": {
60
+ "action_mse_to_best": 0.30881268131689954,
61
+ "expert_success_rate": 0.9375,
62
+ "num_groups": 208,
63
+ "oracle_success_rate": 0.9471153846153846,
64
+ "policy_expert_regret": 1.0182457184150384,
65
+ "policy_oracle_regret": 1.031668585841544,
66
+ "policy_rollout_progress": 0.5859263594745874,
67
+ "policy_rollout_success_rate": 0.3076923076923077,
68
+ "restore_max_error": 4.76837158203125e-07
69
+ },
70
+ "PullCube-v1": {
71
+ "action_mse_to_best": 0.623021765382259,
72
+ "expert_success_rate": 0.19480519480519481,
73
+ "num_groups": 77,
74
+ "oracle_success_rate": 0.36363636363636365,
75
+ "policy_expert_regret": 0.21665887307230527,
76
+ "policy_oracle_regret": 0.38748751849226365,
77
+ "policy_rollout_progress": 0.35096192981612373,
78
+ "policy_rollout_success_rate": 0.24675324675324675,
79
+ "restore_max_error": 4.76837158203125e-07
80
+ },
81
+ "PushCube-v1": {
82
+ "action_mse_to_best": 0.3740823645778793,
83
+ "expert_success_rate": 0.8617021276595744,
84
+ "num_groups": 94,
85
+ "oracle_success_rate": 0.9893617021276596,
86
+ "policy_expert_regret": 0.5533106022692741,
87
+ "policy_oracle_regret": 0.6498078486703812,
88
+ "policy_rollout_progress": 0.6906176832445124,
89
+ "policy_rollout_success_rate": 0.6595744680851063,
90
+ "restore_max_error": 4.76837158203125e-07
91
+ },
92
+ "StackCube-v1": {
93
+ "action_mse_to_best": 0.49294807546480435,
94
+ "expert_success_rate": 0.6767676767676768,
95
+ "num_groups": 99,
96
+ "oracle_success_rate": 0.8585858585858586,
97
+ "policy_expert_regret": 0.9980843377233756,
98
+ "policy_oracle_regret": 1.1726798326379122,
99
+ "policy_rollout_progress": 0.4361087022104649,
100
+ "policy_rollout_success_rate": 0.21212121212121213,
101
+ "restore_max_error": 3.948807716369629e-07
102
+ }
103
+ }
104
+ },
105
+ {
106
+ "seed": 1,
107
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_scale0p25_type_success0p25.json",
108
+ "num_groups": 575,
109
+ "selection_mode": "retrieval_residual",
110
+ "num_candidates": 16,
111
+ "candidate_sigma": 0.0,
112
+ "field_optim_steps": 0,
113
+ "field_optim_step_size": 0.0,
114
+ "field_optim_trust_radius": 0.0,
115
+ "field_optim_l2_penalty": 0.0,
116
+ "retrieval_neighbors": 1,
117
+ "retrieval_metric": "raw",
118
+ "retrieval_type_min_success": 0.25,
119
+ "retrieval_residual_scale": 0.25,
120
+ "policy_rollout_success_rate": 0.3165217391304348,
121
+ "policy_rollout_progress": 0.5491900552825435,
122
+ "oracle_success_rate": 0.8695652173913043,
123
+ "action_mse_to_best": 0.4010985842379539,
124
+ "best_policy_val": {
125
+ "bc_loss": 0.1315989200439718,
126
+ "field_effect_loss": 0.010110370814800262,
127
+ "field_potential_loss": 0.31546536915832096,
128
+ "field_preference_loss": 0.4960637440284093,
129
+ "lattice_edges": 3833.3333333333335,
130
+ "progress_mae": 0.19023226698239645,
131
+ "rank_acc": 0.8255469501018524,
132
+ "rank_loss": 0.4960531195004781,
133
+ "regret_mae": 0.42156532737943864,
134
+ "success_accuracy": 0.8807473679383596,
135
+ "total_loss": 1.561984618504842
136
+ },
137
+ "per_task": {
138
+ "LiftPegUpright-v1": {
139
+ "action_mse_to_best": 0.3345377698656073,
140
+ "expert_success_rate": 0.8584070796460177,
141
+ "num_groups": 113,
142
+ "oracle_success_rate": 0.9380530973451328,
143
+ "policy_expert_regret": 1.006345159316485,
144
+ "policy_oracle_regret": 1.0856339500277443,
145
+ "policy_rollout_progress": 0.6228861411852119,
146
+ "policy_rollout_success_rate": 0.21238938053097345,
147
+ "restore_max_error": 4.76837158203125e-07
148
+ },
149
+ "PickCube-v1": {
150
+ "action_mse_to_best": 0.33102217217421404,
151
+ "expert_success_rate": 0.9402173913043478,
152
+ "num_groups": 184,
153
+ "oracle_success_rate": 0.9456521739130435,
154
+ "policy_expert_regret": 1.1590834624771758,
155
+ "policy_oracle_regret": 1.159107466952106,
156
+ "policy_rollout_progress": 0.538196094460664,
157
+ "policy_rollout_success_rate": 0.23369565217391305,
158
+ "restore_max_error": 4.76837158203125e-07
159
+ },
160
+ "PullCube-v1": {
161
+ "action_mse_to_best": 0.5778321055205244,
162
+ "expert_success_rate": 0.25,
163
+ "num_groups": 76,
164
+ "oracle_success_rate": 0.40789473684210525,
165
+ "policy_expert_regret": 0.33867529622818293,
166
+ "policy_oracle_regret": 0.5427592811909946,
167
+ "policy_rollout_progress": 0.2870815945789218,
168
+ "policy_rollout_success_rate": 0.17105263157894737,
169
+ "restore_max_error": 4.76837158203125e-07
170
+ },
171
+ "PushCube-v1": {
172
+ "action_mse_to_best": 0.37684367295052557,
173
+ "expert_success_rate": 0.8198198198198198,
174
+ "num_groups": 111,
175
+ "oracle_success_rate": 1.0,
176
+ "policy_expert_regret": 0.3626773151996973,
177
+ "policy_oracle_regret": 0.4123242026245272,
178
+ "policy_rollout_progress": 0.803892013591689,
179
+ "policy_rollout_success_rate": 0.7837837837837838,
180
+ "restore_max_error": 4.76837158203125e-07
181
+ },
182
+ "StackCube-v1": {
183
+ "action_mse_to_best": 0.5074280279657835,
184
+ "expert_success_rate": 0.6923076923076923,
185
+ "num_groups": 91,
186
+ "oracle_success_rate": 0.8571428571428571,
187
+ "policy_expert_regret": 1.1193383229630334,
188
+ "policy_oracle_regret": 1.291486077092506,
189
+ "policy_rollout_progress": 0.3881302390779768,
190
+ "policy_rollout_success_rate": 0.16483516483516483,
191
+ "restore_max_error": 3.948807716369629e-07
192
+ }
193
+ }
194
+ },
195
+ {
196
+ "seed": 2,
197
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_scale0p25_type_success0p25.json",
198
+ "num_groups": 575,
199
+ "selection_mode": "retrieval_residual",
200
+ "num_candidates": 16,
201
+ "candidate_sigma": 0.0,
202
+ "field_optim_steps": 0,
203
+ "field_optim_step_size": 0.0,
204
+ "field_optim_trust_radius": 0.0,
205
+ "field_optim_l2_penalty": 0.0,
206
+ "retrieval_neighbors": 1,
207
+ "retrieval_metric": "raw",
208
+ "retrieval_type_min_success": 0.25,
209
+ "retrieval_residual_scale": 0.25,
210
+ "policy_rollout_success_rate": 0.34608695652173915,
211
+ "policy_rollout_progress": 0.5652271221124607,
212
+ "oracle_success_rate": 0.8765217391304347,
213
+ "action_mse_to_best": 0.43034302248174083,
214
+ "best_policy_val": {
215
+ "bc_loss": 0.11367896075050037,
216
+ "field_effect_loss": 0.009670218582161598,
217
+ "field_potential_loss": 0.2641640139950646,
218
+ "field_preference_loss": 0.5130490180518892,
219
+ "lattice_edges": 3833.3333333333335,
220
+ "progress_mae": 0.2021729110015763,
221
+ "rank_acc": 0.8333857821093665,
222
+ "rank_loss": 0.5130119257503085,
223
+ "regret_mae": 0.3958987047274907,
224
+ "success_accuracy": 0.8680730561415354,
225
+ "total_loss": 1.4394984311527677
226
+ },
227
+ "per_task": {
228
+ "LiftPegUpright-v1": {
229
+ "action_mse_to_best": 0.3580305190165139,
230
+ "expert_success_rate": 0.8229166666666666,
231
+ "num_groups": 96,
232
+ "oracle_success_rate": 0.9270833333333334,
233
+ "policy_expert_regret": 0.8474036318560442,
234
+ "policy_oracle_regret": 0.9536635428667068,
235
+ "policy_rollout_progress": 0.6466116470595201,
236
+ "policy_rollout_success_rate": 0.3333333333333333,
237
+ "restore_max_error": 3.5762786865234375e-07
238
+ },
239
+ "PickCube-v1": {
240
+ "action_mse_to_best": 0.34562643173367086,
241
+ "expert_success_rate": 0.9444444444444444,
242
+ "num_groups": 198,
243
+ "oracle_success_rate": 0.9595959595959596,
244
+ "policy_expert_regret": 1.0567232194083809,
245
+ "policy_oracle_regret": 1.0638066699370892,
246
+ "policy_rollout_progress": 0.5980631956887065,
247
+ "policy_rollout_success_rate": 0.29292929292929293,
248
+ "restore_max_error": 4.76837158203125e-07
249
+ },
250
+ "PullCube-v1": {
251
+ "action_mse_to_best": 0.6627736421922843,
252
+ "expert_success_rate": 0.24444444444444444,
253
+ "num_groups": 90,
254
+ "oracle_success_rate": 0.4666666666666667,
255
+ "policy_expert_regret": 0.2764986334575547,
256
+ "policy_oracle_regret": 0.5040211054599948,
257
+ "policy_rollout_progress": 0.3299036862535609,
258
+ "policy_rollout_success_rate": 0.2222222222222222,
259
+ "restore_max_error": 4.0978193283081055e-07
260
+ },
261
+ "PushCube-v1": {
262
+ "action_mse_to_best": 0.3916822287558329,
263
+ "expert_success_rate": 0.8514851485148515,
264
+ "num_groups": 101,
265
+ "oracle_success_rate": 1.0,
266
+ "policy_expert_regret": 0.4015513146572774,
267
+ "policy_oracle_regret": 0.4742009368273291,
268
+ "policy_rollout_progress": 0.7733238156479184,
269
+ "policy_rollout_success_rate": 0.7524752475247525,
270
+ "restore_max_error": 4.76837158203125e-07
271
+ },
272
+ "StackCube-v1": {
273
+ "action_mse_to_best": 0.50480824129449,
274
+ "expert_success_rate": 0.7666666666666667,
275
+ "num_groups": 90,
276
+ "oracle_success_rate": 0.9111111111111111,
277
+ "policy_expert_regret": 1.1974563411540455,
278
+ "policy_oracle_regret": 1.3337924069828457,
279
+ "policy_rollout_progress": 0.4079703023036321,
280
+ "policy_rollout_success_rate": 0.14444444444444443,
281
+ "restore_max_error": 4.76837158203125e-07
282
+ }
283
+ }
284
+ }
285
+ ]
286
+ }
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p25_type_success0p25_summary.md ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # h=16 Best-Policy Checkpoint Rollout
2
+
3
+ Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
4
+ Objective: `near_miss_policy_bc5`
5
+ Result file: `policy_rollout_retrieval_residual_scale0p25_type_success0p25.json`
6
+ Completed seeds: 3
7
+ Baseline h=4 policy success: 29.67%
8
+ Baseline h=16 rank-checkpoint success: 29.74%
9
+
10
+ Mean success: 32.93% +/- 1.52%
11
+ Gain vs h=16 rank checkpoint: +3.19%
12
+ Mean progress: 55.24%
13
+ Mean action MSE to best: 0.409
14
+
15
+ | seed | mode | k | retrieval K | retrieval metric | min type success | residual scale | sigma | opt steps | trust | success | progress | oracle | action MSE |
16
+ |---:|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
17
+ | 0 | retrieval_residual | 16 | 1 | raw | 0.25 | 0.25 | 0.00 | 0 | 0.00 | 32.52% | 54.27% | 85.74% | 0.396 |
18
+ | 1 | retrieval_residual | 16 | 1 | raw | 0.25 | 0.25 | 0.00 | 0 | 0.00 | 31.65% | 54.92% | 86.96% | 0.401 |
19
+ | 2 | retrieval_residual | 16 | 1 | raw | 0.25 | 0.25 | 0.00 | 0 | 0.00 | 34.61% | 56.52% | 87.65% | 0.430 |
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p50_type_success0p10_summary.json ADDED
@@ -0,0 +1,286 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
3
+ "objective": "near_miss_policy_bc5",
4
+ "out_name": "policy_rollout_retrieval_residual_scale0p50_type_success0p10.json",
5
+ "num_completed": 3,
6
+ "baseline_h4_policy_success": 0.2967,
7
+ "baseline_h16_rank_checkpoint_success": 0.29739130434782607,
8
+ "mean_success": 0.3333333333333333,
9
+ "std_success": 0.00821880978478714,
10
+ "mean_progress": 0.5528276873363749,
11
+ "mean_action_mse_to_best": 0.4331563813282528,
12
+ "gain_vs_h4": 0.036633333333333296,
13
+ "gain_vs_h16_rank_checkpoint": 0.035942028985507246,
14
+ "rows": [
15
+ {
16
+ "seed": 0,
17
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_scale0p50_type_success0p10.json",
18
+ "num_groups": 575,
19
+ "selection_mode": "retrieval_residual",
20
+ "num_candidates": 16,
21
+ "candidate_sigma": 0.0,
22
+ "field_optim_steps": 0,
23
+ "field_optim_step_size": 0.0,
24
+ "field_optim_trust_radius": 0.0,
25
+ "field_optim_l2_penalty": 0.0,
26
+ "retrieval_neighbors": 1,
27
+ "retrieval_metric": "raw",
28
+ "retrieval_type_min_success": 0.1,
29
+ "retrieval_residual_scale": 0.5,
30
+ "policy_rollout_success_rate": 0.33043478260869563,
31
+ "policy_rollout_progress": 0.544471482436942,
32
+ "oracle_success_rate": 0.8573913043478261,
33
+ "action_mse_to_best": 0.4129274774707206,
34
+ "best_policy_val": {
35
+ "bc_loss": 0.13721593966086706,
36
+ "field_effect_loss": 0.009290305380192068,
37
+ "field_potential_loss": 0.2666468388504452,
38
+ "field_preference_loss": 0.5130573478009965,
39
+ "lattice_edges": 3833.3333333333335,
40
+ "progress_mae": 0.1933159919248687,
41
+ "rank_acc": 0.8265031774838766,
42
+ "rank_loss": 0.5130523675017886,
43
+ "regret_mae": 0.3756548762321472,
44
+ "success_accuracy": 0.8773836526605818,
45
+ "total_loss": 1.5581054819954767
46
+ },
47
+ "per_task": {
48
+ "LiftPegUpright-v1": {
49
+ "action_mse_to_best": 0.33314846369159434,
50
+ "expert_success_rate": 0.8865979381443299,
51
+ "num_groups": 97,
52
+ "oracle_success_rate": 0.9278350515463918,
53
+ "policy_expert_regret": 1.0792845781009222,
54
+ "policy_oracle_regret": 1.105700055809365,
55
+ "policy_rollout_progress": 0.5705123155080166,
56
+ "policy_rollout_success_rate": 0.2268041237113402,
57
+ "restore_max_error": 4.76837158203125e-07
58
+ },
59
+ "PickCube-v1": {
60
+ "action_mse_to_best": 0.3205748214744605,
61
+ "expert_success_rate": 0.9375,
62
+ "num_groups": 208,
63
+ "oracle_success_rate": 0.9471153846153846,
64
+ "policy_expert_regret": 1.0125951060147669,
65
+ "policy_oracle_regret": 1.0260179734412724,
66
+ "policy_rollout_progress": 0.5887099993504727,
67
+ "policy_rollout_success_rate": 0.3173076923076923,
68
+ "restore_max_error": 4.76837158203125e-07
69
+ },
70
+ "PullCube-v1": {
71
+ "action_mse_to_best": 0.6671761597518797,
72
+ "expert_success_rate": 0.19480519480519481,
73
+ "num_groups": 77,
74
+ "oracle_success_rate": 0.36363636363636365,
75
+ "policy_expert_regret": 0.24359342272018458,
76
+ "policy_oracle_regret": 0.4621977220204743,
77
+ "policy_rollout_progress": 0.3145493839371514,
78
+ "policy_rollout_success_rate": 0.2077922077922078,
79
+ "restore_max_error": 4.76837158203125e-07
80
+ },
81
+ "PushCube-v1": {
82
+ "action_mse_to_best": 0.3714648786773707,
83
+ "expert_success_rate": 0.8617021276595744,
84
+ "num_groups": 94,
85
+ "oracle_success_rate": 0.9893617021276596,
86
+ "policy_expert_regret": 0.5719942746961371,
87
+ "policy_oracle_regret": 0.6495450359392674,
88
+ "policy_rollout_progress": 0.6908804959756263,
89
+ "policy_rollout_success_rate": 0.6595744680851063,
90
+ "restore_max_error": 4.76837158203125e-07
91
+ },
92
+ "StackCube-v1": {
93
+ "action_mse_to_best": 0.5267482101446902,
94
+ "expert_success_rate": 0.6767676767676768,
95
+ "num_groups": 99,
96
+ "oracle_success_rate": 0.8585858585858586,
97
+ "policy_expert_regret": 0.9495171221217724,
98
+ "policy_oracle_regret": 1.112660668715082,
99
+ "policy_rollout_progress": 0.4658248358302646,
100
+ "policy_rollout_success_rate": 0.24242424242424243,
101
+ "restore_max_error": 3.948807716369629e-07
102
+ }
103
+ }
104
+ },
105
+ {
106
+ "seed": 1,
107
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_scale0p50_type_success0p10.json",
108
+ "num_groups": 575,
109
+ "selection_mode": "retrieval_residual",
110
+ "num_candidates": 16,
111
+ "candidate_sigma": 0.0,
112
+ "field_optim_steps": 0,
113
+ "field_optim_step_size": 0.0,
114
+ "field_optim_trust_radius": 0.0,
115
+ "field_optim_l2_penalty": 0.0,
116
+ "retrieval_neighbors": 1,
117
+ "retrieval_metric": "raw",
118
+ "retrieval_type_min_success": 0.1,
119
+ "retrieval_residual_scale": 0.5,
120
+ "policy_rollout_success_rate": 0.3269565217391304,
121
+ "policy_rollout_progress": 0.5530163816378816,
122
+ "oracle_success_rate": 0.8695652173913043,
123
+ "action_mse_to_best": 0.42286321135642735,
124
+ "best_policy_val": {
125
+ "bc_loss": 0.1315989200439718,
126
+ "field_effect_loss": 0.010110370814800262,
127
+ "field_potential_loss": 0.31546536915832096,
128
+ "field_preference_loss": 0.4960637440284093,
129
+ "lattice_edges": 3833.3333333333335,
130
+ "progress_mae": 0.19023226698239645,
131
+ "rank_acc": 0.8255469501018524,
132
+ "rank_loss": 0.4960531195004781,
133
+ "regret_mae": 0.42156532737943864,
134
+ "success_accuracy": 0.8807473679383596,
135
+ "total_loss": 1.561984618504842
136
+ },
137
+ "per_task": {
138
+ "LiftPegUpright-v1": {
139
+ "action_mse_to_best": 0.33098896499723196,
140
+ "expert_success_rate": 0.8584070796460177,
141
+ "num_groups": 113,
142
+ "oracle_success_rate": 0.9380530973451328,
143
+ "policy_expert_regret": 0.998058971973647,
144
+ "policy_oracle_regret": 1.0891584641901793,
145
+ "policy_rollout_progress": 0.619361627022777,
146
+ "policy_rollout_success_rate": 0.21238938053097345,
147
+ "restore_max_error": 4.76837158203125e-07
148
+ },
149
+ "PickCube-v1": {
150
+ "action_mse_to_best": 0.3595517423692281,
151
+ "expert_success_rate": 0.9402173913043478,
152
+ "num_groups": 184,
153
+ "oracle_success_rate": 0.9456521739130435,
154
+ "policy_expert_regret": 1.1190804282618363,
155
+ "policy_oracle_regret": 1.1266173332291858,
156
+ "policy_rollout_progress": 0.5457061722864518,
157
+ "policy_rollout_success_rate": 0.266304347826087,
158
+ "restore_max_error": 4.76837158203125e-07
159
+ },
160
+ "PullCube-v1": {
161
+ "action_mse_to_best": 0.6149560018981758,
162
+ "expert_success_rate": 0.25,
163
+ "num_groups": 76,
164
+ "oracle_success_rate": 0.40789473684210525,
165
+ "policy_expert_regret": 0.27304727383154004,
166
+ "policy_oracle_regret": 0.5447679004680953,
167
+ "policy_rollout_progress": 0.27486991495090096,
168
+ "policy_rollout_success_rate": 0.15789473684210525,
169
+ "restore_max_error": 4.76837158203125e-07
170
+ },
171
+ "PushCube-v1": {
172
+ "action_mse_to_best": 0.39232181858372045,
173
+ "expert_success_rate": 0.8198198198198198,
174
+ "num_groups": 111,
175
+ "oracle_success_rate": 1.0,
176
+ "policy_expert_regret": 0.34771036054636983,
177
+ "policy_oracle_regret": 0.38119086757436527,
178
+ "policy_rollout_progress": 0.8170073306238329,
179
+ "policy_rollout_success_rate": 0.8018018018018018,
180
+ "restore_max_error": 4.76837158203125e-07
181
+ },
182
+ "StackCube-v1": {
183
+ "action_mse_to_best": 0.5417878558490794,
184
+ "expert_success_rate": 0.6923076923076923,
185
+ "num_groups": 91,
186
+ "oracle_success_rate": 0.8571428571428571,
187
+ "policy_expert_regret": 1.122757653777416,
188
+ "policy_oracle_regret": 1.2949054079068887,
189
+ "policy_rollout_progress": 0.3956999192526052,
190
+ "policy_rollout_success_rate": 0.15384615384615385,
191
+ "restore_max_error": 3.948807716369629e-07
192
+ }
193
+ }
194
+ },
195
+ {
196
+ "seed": 2,
197
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_scale0p50_type_success0p10.json",
198
+ "num_groups": 575,
199
+ "selection_mode": "retrieval_residual",
200
+ "num_candidates": 16,
201
+ "candidate_sigma": 0.0,
202
+ "field_optim_steps": 0,
203
+ "field_optim_step_size": 0.0,
204
+ "field_optim_trust_radius": 0.0,
205
+ "field_optim_l2_penalty": 0.0,
206
+ "retrieval_neighbors": 1,
207
+ "retrieval_metric": "raw",
208
+ "retrieval_type_min_success": 0.1,
209
+ "retrieval_residual_scale": 0.5,
210
+ "policy_rollout_success_rate": 0.3426086956521739,
211
+ "policy_rollout_progress": 0.560995197934301,
212
+ "oracle_success_rate": 0.8765217391304347,
213
+ "action_mse_to_best": 0.46367845515761036,
214
+ "best_policy_val": {
215
+ "bc_loss": 0.11367896075050037,
216
+ "field_effect_loss": 0.009670218582161598,
217
+ "field_potential_loss": 0.2641640139950646,
218
+ "field_preference_loss": 0.5130490180518892,
219
+ "lattice_edges": 3833.3333333333335,
220
+ "progress_mae": 0.2021729110015763,
221
+ "rank_acc": 0.8333857821093665,
222
+ "rank_loss": 0.5130119257503085,
223
+ "regret_mae": 0.3958987047274907,
224
+ "success_accuracy": 0.8680730561415354,
225
+ "total_loss": 1.4394984311527677
226
+ },
227
+ "per_task": {
228
+ "LiftPegUpright-v1": {
229
+ "action_mse_to_best": 0.36153392906514153,
230
+ "expert_success_rate": 0.8229166666666666,
231
+ "num_groups": 96,
232
+ "oracle_success_rate": 0.9270833333333334,
233
+ "policy_expert_regret": 0.8829973929872116,
234
+ "policy_oracle_regret": 0.9760098507006963,
235
+ "policy_rollout_progress": 0.6346820058921973,
236
+ "policy_rollout_success_rate": 0.3229166666666667,
237
+ "restore_max_error": 3.5762786865234375e-07
238
+ },
239
+ "PickCube-v1": {
240
+ "action_mse_to_best": 0.37652896986239487,
241
+ "expert_success_rate": 0.9444444444444444,
242
+ "num_groups": 198,
243
+ "oracle_success_rate": 0.9595959595959596,
244
+ "policy_expert_regret": 1.0838303450457376,
245
+ "policy_oracle_regret": 1.090913795574446,
246
+ "policy_rollout_progress": 0.5840688492688868,
247
+ "policy_rollout_success_rate": 0.2727272727272727,
248
+ "restore_max_error": 4.76837158203125e-07
249
+ },
250
+ "PullCube-v1": {
251
+ "action_mse_to_best": 0.7301618857516183,
252
+ "expert_success_rate": 0.24444444444444444,
253
+ "num_groups": 90,
254
+ "oracle_success_rate": 0.4666666666666667,
255
+ "policy_expert_regret": 0.30010166329642135,
256
+ "policy_oracle_regret": 0.5302090961693062,
257
+ "policy_rollout_progress": 0.31182427391823797,
258
+ "policy_rollout_success_rate": 0.2111111111111111,
259
+ "restore_max_error": 4.0978193283081055e-07
260
+ },
261
+ "PushCube-v1": {
262
+ "action_mse_to_best": 0.4178345319215614,
263
+ "expert_success_rate": 0.8514851485148515,
264
+ "num_groups": 101,
265
+ "oracle_success_rate": 1.0,
266
+ "policy_expert_regret": 0.38219682946063505,
267
+ "policy_oracle_regret": 0.4192075215943969,
268
+ "policy_rollout_progress": 0.7986142605838209,
269
+ "policy_rollout_success_rate": 0.7821782178217822,
270
+ "restore_max_error": 4.76837158203125e-07
271
+ },
272
+ "StackCube-v1": {
273
+ "action_mse_to_best": 0.5493251227877206,
274
+ "expert_success_rate": 0.7666666666666667,
275
+ "num_groups": 90,
276
+ "oracle_success_rate": 0.9111111111111111,
277
+ "policy_expert_regret": 1.15032865587208,
278
+ "policy_oracle_regret": 1.3123159415192074,
279
+ "policy_rollout_progress": 0.41414343466361364,
280
+ "policy_rollout_success_rate": 0.15555555555555556,
281
+ "restore_max_error": 4.76837158203125e-07
282
+ }
283
+ }
284
+ }
285
+ ]
286
+ }
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p50_type_success0p10_summary.md ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # h=16 Best-Policy Checkpoint Rollout
2
+
3
+ Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
4
+ Objective: `near_miss_policy_bc5`
5
+ Result file: `policy_rollout_retrieval_residual_scale0p50_type_success0p10.json`
6
+ Completed seeds: 3
7
+ Baseline h=4 policy success: 29.67%
8
+ Baseline h=16 rank-checkpoint success: 29.74%
9
+
10
+ Mean success: 33.33% +/- 0.82%
11
+ Gain vs h=16 rank checkpoint: +3.59%
12
+ Mean progress: 55.28%
13
+ Mean action MSE to best: 0.433
14
+
15
+ | seed | mode | k | retrieval K | retrieval metric | min type success | residual scale | sigma | opt steps | trust | success | progress | oracle | action MSE |
16
+ |---:|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
17
+ | 0 | retrieval_residual | 16 | 1 | raw | 0.10 | 0.50 | 0.00 | 0 | 0.00 | 33.04% | 54.45% | 85.74% | 0.413 |
18
+ | 1 | retrieval_residual | 16 | 1 | raw | 0.10 | 0.50 | 0.00 | 0 | 0.00 | 32.70% | 55.30% | 86.96% | 0.423 |
19
+ | 2 | retrieval_residual | 16 | 1 | raw | 0.10 | 0.50 | 0.00 | 0 | 0.00 | 34.26% | 56.10% | 87.65% | 0.464 |
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p50_type_success0p25_summary.json ADDED
@@ -0,0 +1,286 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
3
+ "objective": "near_miss_policy_bc5",
4
+ "out_name": "policy_rollout_retrieval_residual_scale0p50_type_success0p25.json",
5
+ "num_completed": 3,
6
+ "baseline_h4_policy_success": 0.2967,
7
+ "baseline_h16_rank_checkpoint_success": 0.29739130434782607,
8
+ "mean_success": 0.3333333333333333,
9
+ "std_success": 0.00821880978478714,
10
+ "mean_progress": 0.5528276873363749,
11
+ "mean_action_mse_to_best": 0.4331563813282528,
12
+ "gain_vs_h4": 0.036633333333333296,
13
+ "gain_vs_h16_rank_checkpoint": 0.035942028985507246,
14
+ "rows": [
15
+ {
16
+ "seed": 0,
17
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_scale0p50_type_success0p25.json",
18
+ "num_groups": 575,
19
+ "selection_mode": "retrieval_residual",
20
+ "num_candidates": 16,
21
+ "candidate_sigma": 0.0,
22
+ "field_optim_steps": 0,
23
+ "field_optim_step_size": 0.0,
24
+ "field_optim_trust_radius": 0.0,
25
+ "field_optim_l2_penalty": 0.0,
26
+ "retrieval_neighbors": 1,
27
+ "retrieval_metric": "raw",
28
+ "retrieval_type_min_success": 0.25,
29
+ "retrieval_residual_scale": 0.5,
30
+ "policy_rollout_success_rate": 0.33043478260869563,
31
+ "policy_rollout_progress": 0.544471482436942,
32
+ "oracle_success_rate": 0.8573913043478261,
33
+ "action_mse_to_best": 0.4129274774707206,
34
+ "best_policy_val": {
35
+ "bc_loss": 0.13721593966086706,
36
+ "field_effect_loss": 0.009290305380192068,
37
+ "field_potential_loss": 0.2666468388504452,
38
+ "field_preference_loss": 0.5130573478009965,
39
+ "lattice_edges": 3833.3333333333335,
40
+ "progress_mae": 0.1933159919248687,
41
+ "rank_acc": 0.8265031774838766,
42
+ "rank_loss": 0.5130523675017886,
43
+ "regret_mae": 0.3756548762321472,
44
+ "success_accuracy": 0.8773836526605818,
45
+ "total_loss": 1.5581054819954767
46
+ },
47
+ "per_task": {
48
+ "LiftPegUpright-v1": {
49
+ "action_mse_to_best": 0.33314846369159434,
50
+ "expert_success_rate": 0.8865979381443299,
51
+ "num_groups": 97,
52
+ "oracle_success_rate": 0.9278350515463918,
53
+ "policy_expert_regret": 1.0792845781009222,
54
+ "policy_oracle_regret": 1.105700055809365,
55
+ "policy_rollout_progress": 0.5705123155080166,
56
+ "policy_rollout_success_rate": 0.2268041237113402,
57
+ "restore_max_error": 4.76837158203125e-07
58
+ },
59
+ "PickCube-v1": {
60
+ "action_mse_to_best": 0.3205748214744605,
61
+ "expert_success_rate": 0.9375,
62
+ "num_groups": 208,
63
+ "oracle_success_rate": 0.9471153846153846,
64
+ "policy_expert_regret": 1.0125951060147669,
65
+ "policy_oracle_regret": 1.0260179734412724,
66
+ "policy_rollout_progress": 0.5887099993504727,
67
+ "policy_rollout_success_rate": 0.3173076923076923,
68
+ "restore_max_error": 4.76837158203125e-07
69
+ },
70
+ "PullCube-v1": {
71
+ "action_mse_to_best": 0.6671761597518797,
72
+ "expert_success_rate": 0.19480519480519481,
73
+ "num_groups": 77,
74
+ "oracle_success_rate": 0.36363636363636365,
75
+ "policy_expert_regret": 0.24359342272018458,
76
+ "policy_oracle_regret": 0.4621977220204743,
77
+ "policy_rollout_progress": 0.3145493839371514,
78
+ "policy_rollout_success_rate": 0.2077922077922078,
79
+ "restore_max_error": 4.76837158203125e-07
80
+ },
81
+ "PushCube-v1": {
82
+ "action_mse_to_best": 0.3714648786773707,
83
+ "expert_success_rate": 0.8617021276595744,
84
+ "num_groups": 94,
85
+ "oracle_success_rate": 0.9893617021276596,
86
+ "policy_expert_regret": 0.5719942746961371,
87
+ "policy_oracle_regret": 0.6495450359392674,
88
+ "policy_rollout_progress": 0.6908804959756263,
89
+ "policy_rollout_success_rate": 0.6595744680851063,
90
+ "restore_max_error": 4.76837158203125e-07
91
+ },
92
+ "StackCube-v1": {
93
+ "action_mse_to_best": 0.5267482101446902,
94
+ "expert_success_rate": 0.6767676767676768,
95
+ "num_groups": 99,
96
+ "oracle_success_rate": 0.8585858585858586,
97
+ "policy_expert_regret": 0.9495171221217724,
98
+ "policy_oracle_regret": 1.112660668715082,
99
+ "policy_rollout_progress": 0.4658248358302646,
100
+ "policy_rollout_success_rate": 0.24242424242424243,
101
+ "restore_max_error": 3.948807716369629e-07
102
+ }
103
+ }
104
+ },
105
+ {
106
+ "seed": 1,
107
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_scale0p50_type_success0p25.json",
108
+ "num_groups": 575,
109
+ "selection_mode": "retrieval_residual",
110
+ "num_candidates": 16,
111
+ "candidate_sigma": 0.0,
112
+ "field_optim_steps": 0,
113
+ "field_optim_step_size": 0.0,
114
+ "field_optim_trust_radius": 0.0,
115
+ "field_optim_l2_penalty": 0.0,
116
+ "retrieval_neighbors": 1,
117
+ "retrieval_metric": "raw",
118
+ "retrieval_type_min_success": 0.25,
119
+ "retrieval_residual_scale": 0.5,
120
+ "policy_rollout_success_rate": 0.3269565217391304,
121
+ "policy_rollout_progress": 0.5530163816378816,
122
+ "oracle_success_rate": 0.8695652173913043,
123
+ "action_mse_to_best": 0.42286321135642735,
124
+ "best_policy_val": {
125
+ "bc_loss": 0.1315989200439718,
126
+ "field_effect_loss": 0.010110370814800262,
127
+ "field_potential_loss": 0.31546536915832096,
128
+ "field_preference_loss": 0.4960637440284093,
129
+ "lattice_edges": 3833.3333333333335,
130
+ "progress_mae": 0.19023226698239645,
131
+ "rank_acc": 0.8255469501018524,
132
+ "rank_loss": 0.4960531195004781,
133
+ "regret_mae": 0.42156532737943864,
134
+ "success_accuracy": 0.8807473679383596,
135
+ "total_loss": 1.561984618504842
136
+ },
137
+ "per_task": {
138
+ "LiftPegUpright-v1": {
139
+ "action_mse_to_best": 0.33098896499723196,
140
+ "expert_success_rate": 0.8584070796460177,
141
+ "num_groups": 113,
142
+ "oracle_success_rate": 0.9380530973451328,
143
+ "policy_expert_regret": 0.998058971973647,
144
+ "policy_oracle_regret": 1.0891584641901793,
145
+ "policy_rollout_progress": 0.619361627022777,
146
+ "policy_rollout_success_rate": 0.21238938053097345,
147
+ "restore_max_error": 4.76837158203125e-07
148
+ },
149
+ "PickCube-v1": {
150
+ "action_mse_to_best": 0.3595517423692281,
151
+ "expert_success_rate": 0.9402173913043478,
152
+ "num_groups": 184,
153
+ "oracle_success_rate": 0.9456521739130435,
154
+ "policy_expert_regret": 1.1190804282618363,
155
+ "policy_oracle_regret": 1.1266173332291858,
156
+ "policy_rollout_progress": 0.5457061722864518,
157
+ "policy_rollout_success_rate": 0.266304347826087,
158
+ "restore_max_error": 4.76837158203125e-07
159
+ },
160
+ "PullCube-v1": {
161
+ "action_mse_to_best": 0.6149560018981758,
162
+ "expert_success_rate": 0.25,
163
+ "num_groups": 76,
164
+ "oracle_success_rate": 0.40789473684210525,
165
+ "policy_expert_regret": 0.27304727383154004,
166
+ "policy_oracle_regret": 0.5447679004680953,
167
+ "policy_rollout_progress": 0.27486991495090096,
168
+ "policy_rollout_success_rate": 0.15789473684210525,
169
+ "restore_max_error": 4.76837158203125e-07
170
+ },
171
+ "PushCube-v1": {
172
+ "action_mse_to_best": 0.39232181858372045,
173
+ "expert_success_rate": 0.8198198198198198,
174
+ "num_groups": 111,
175
+ "oracle_success_rate": 1.0,
176
+ "policy_expert_regret": 0.34771036054636983,
177
+ "policy_oracle_regret": 0.38119086757436527,
178
+ "policy_rollout_progress": 0.8170073306238329,
179
+ "policy_rollout_success_rate": 0.8018018018018018,
180
+ "restore_max_error": 4.76837158203125e-07
181
+ },
182
+ "StackCube-v1": {
183
+ "action_mse_to_best": 0.5417878558490794,
184
+ "expert_success_rate": 0.6923076923076923,
185
+ "num_groups": 91,
186
+ "oracle_success_rate": 0.8571428571428571,
187
+ "policy_expert_regret": 1.122757653777416,
188
+ "policy_oracle_regret": 1.2949054079068887,
189
+ "policy_rollout_progress": 0.3956999192526052,
190
+ "policy_rollout_success_rate": 0.15384615384615385,
191
+ "restore_max_error": 3.948807716369629e-07
192
+ }
193
+ }
194
+ },
195
+ {
196
+ "seed": 2,
197
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_scale0p50_type_success0p25.json",
198
+ "num_groups": 575,
199
+ "selection_mode": "retrieval_residual",
200
+ "num_candidates": 16,
201
+ "candidate_sigma": 0.0,
202
+ "field_optim_steps": 0,
203
+ "field_optim_step_size": 0.0,
204
+ "field_optim_trust_radius": 0.0,
205
+ "field_optim_l2_penalty": 0.0,
206
+ "retrieval_neighbors": 1,
207
+ "retrieval_metric": "raw",
208
+ "retrieval_type_min_success": 0.25,
209
+ "retrieval_residual_scale": 0.5,
210
+ "policy_rollout_success_rate": 0.3426086956521739,
211
+ "policy_rollout_progress": 0.560995197934301,
212
+ "oracle_success_rate": 0.8765217391304347,
213
+ "action_mse_to_best": 0.46367845515761036,
214
+ "best_policy_val": {
215
+ "bc_loss": 0.11367896075050037,
216
+ "field_effect_loss": 0.009670218582161598,
217
+ "field_potential_loss": 0.2641640139950646,
218
+ "field_preference_loss": 0.5130490180518892,
219
+ "lattice_edges": 3833.3333333333335,
220
+ "progress_mae": 0.2021729110015763,
221
+ "rank_acc": 0.8333857821093665,
222
+ "rank_loss": 0.5130119257503085,
223
+ "regret_mae": 0.3958987047274907,
224
+ "success_accuracy": 0.8680730561415354,
225
+ "total_loss": 1.4394984311527677
226
+ },
227
+ "per_task": {
228
+ "LiftPegUpright-v1": {
229
+ "action_mse_to_best": 0.36153392906514153,
230
+ "expert_success_rate": 0.8229166666666666,
231
+ "num_groups": 96,
232
+ "oracle_success_rate": 0.9270833333333334,
233
+ "policy_expert_regret": 0.8829973929872116,
234
+ "policy_oracle_regret": 0.9760098507006963,
235
+ "policy_rollout_progress": 0.6346820058921973,
236
+ "policy_rollout_success_rate": 0.3229166666666667,
237
+ "restore_max_error": 3.5762786865234375e-07
238
+ },
239
+ "PickCube-v1": {
240
+ "action_mse_to_best": 0.37652896986239487,
241
+ "expert_success_rate": 0.9444444444444444,
242
+ "num_groups": 198,
243
+ "oracle_success_rate": 0.9595959595959596,
244
+ "policy_expert_regret": 1.0838303450457376,
245
+ "policy_oracle_regret": 1.090913795574446,
246
+ "policy_rollout_progress": 0.5840688492688868,
247
+ "policy_rollout_success_rate": 0.2727272727272727,
248
+ "restore_max_error": 4.76837158203125e-07
249
+ },
250
+ "PullCube-v1": {
251
+ "action_mse_to_best": 0.7301618857516183,
252
+ "expert_success_rate": 0.24444444444444444,
253
+ "num_groups": 90,
254
+ "oracle_success_rate": 0.4666666666666667,
255
+ "policy_expert_regret": 0.30010166329642135,
256
+ "policy_oracle_regret": 0.5302090961693062,
257
+ "policy_rollout_progress": 0.31182427391823797,
258
+ "policy_rollout_success_rate": 0.2111111111111111,
259
+ "restore_max_error": 4.0978193283081055e-07
260
+ },
261
+ "PushCube-v1": {
262
+ "action_mse_to_best": 0.4178345319215614,
263
+ "expert_success_rate": 0.8514851485148515,
264
+ "num_groups": 101,
265
+ "oracle_success_rate": 1.0,
266
+ "policy_expert_regret": 0.38219682946063505,
267
+ "policy_oracle_regret": 0.4192075215943969,
268
+ "policy_rollout_progress": 0.7986142605838209,
269
+ "policy_rollout_success_rate": 0.7821782178217822,
270
+ "restore_max_error": 4.76837158203125e-07
271
+ },
272
+ "StackCube-v1": {
273
+ "action_mse_to_best": 0.5493251227877206,
274
+ "expert_success_rate": 0.7666666666666667,
275
+ "num_groups": 90,
276
+ "oracle_success_rate": 0.9111111111111111,
277
+ "policy_expert_regret": 1.15032865587208,
278
+ "policy_oracle_regret": 1.3123159415192074,
279
+ "policy_rollout_progress": 0.41414343466361364,
280
+ "policy_rollout_success_rate": 0.15555555555555556,
281
+ "restore_max_error": 4.76837158203125e-07
282
+ }
283
+ }
284
+ }
285
+ ]
286
+ }
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p50_type_success0p25_summary.md ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # h=16 Best-Policy Checkpoint Rollout
2
+
3
+ Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
4
+ Objective: `near_miss_policy_bc5`
5
+ Result file: `policy_rollout_retrieval_residual_scale0p50_type_success0p25.json`
6
+ Completed seeds: 3
7
+ Baseline h=4 policy success: 29.67%
8
+ Baseline h=16 rank-checkpoint success: 29.74%
9
+
10
+ Mean success: 33.33% +/- 0.82%
11
+ Gain vs h=16 rank checkpoint: +3.59%
12
+ Mean progress: 55.28%
13
+ Mean action MSE to best: 0.433
14
+
15
+ | seed | mode | k | retrieval K | retrieval metric | min type success | residual scale | sigma | opt steps | trust | success | progress | oracle | action MSE |
16
+ |---:|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
17
+ | 0 | retrieval_residual | 16 | 1 | raw | 0.25 | 0.50 | 0.00 | 0 | 0.00 | 33.04% | 54.45% | 85.74% | 0.413 |
18
+ | 1 | retrieval_residual | 16 | 1 | raw | 0.25 | 0.50 | 0.00 | 0 | 0.00 | 32.70% | 55.30% | 86.96% | 0.423 |
19
+ | 2 | retrieval_residual | 16 | 1 | raw | 0.25 | 0.50 | 0.00 | 0 | 0.00 | 34.26% | 56.10% | 87.65% | 0.464 |
results/paper_story_memo.md CHANGED
@@ -114,6 +114,12 @@ Last checked: `2026-06-28 05:47 UTC`.
114
  - `14859297`/`14859298`: active train-family reliability eval/summary, scale
115
  `0.25`, minimum train success `0.25`.
116
  - `14859299`: rebuild `paper_table_status.*` after reliability-prior summaries.
 
 
 
 
 
 
117
 
118
  ## Decision Rule For Masked Residual Jobs
119
 
 
114
  - `14859297`/`14859298`: active train-family reliability eval/summary, scale
115
  `0.25`, minimum train success `0.25`.
116
  - `14859299`: rebuild `paper_table_status.*` after reliability-prior summaries.
117
+ - `14859398`/`14859399`: active train-family reliability eval/summary, scale
118
+ `0.50`, minimum train success `0.50`.
119
+ - `14859400`/`14859401`: active train-family reliability eval/summary, scale
120
+ `0.50`, minimum train success `0.75`.
121
+ - `14859402`: rebuild `paper_table_status.*` after high-threshold reliability
122
+ summaries.
123
 
124
  ## Decision Rule For Masked Residual Jobs
125
 
results/paper_table_status.json CHANGED
@@ -460,6 +460,44 @@
460
  "best_config": null,
461
  "gain_vs_h16_policy": 0.035942028985507246
462
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
463
  {
464
  "key": "retrieval_residual_scale025_type_success025",
465
  "label": "Train-state residual retrieval, scale 0.25, train family success >= 0.25",
 
460
  "best_config": null,
461
  "gain_vs_h16_policy": 0.035942028985507246
462
  },
463
+ {
464
+ "key": "retrieval_residual_scale050_type_success050",
465
+ "label": "Train-state residual retrieval, scale 0.50, train family success >= 0.50",
466
+ "path": "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p50_type_success0p50_summary.json",
467
+ "clean_deployment": "yes",
468
+ "same_state_proposals": "no",
469
+ "expert_proposal": "no",
470
+ "story_role": "train-split residual family reliability prior",
471
+ "fallback_success": null,
472
+ "pending_job": "14859398/14859399",
473
+ "path_exists": false,
474
+ "status": "pending",
475
+ "success": null,
476
+ "std_success": null,
477
+ "completed_seeds": null,
478
+ "num_completed": null,
479
+ "best_config": null,
480
+ "gain_vs_h16_policy": null
481
+ },
482
+ {
483
+ "key": "retrieval_residual_scale050_type_success075",
484
+ "label": "Train-state residual retrieval, scale 0.50, train family success >= 0.75",
485
+ "path": "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p50_type_success0p75_summary.json",
486
+ "clean_deployment": "yes",
487
+ "same_state_proposals": "no",
488
+ "expert_proposal": "no",
489
+ "story_role": "train-split residual family reliability prior",
490
+ "fallback_success": null,
491
+ "pending_job": "14859400/14859401",
492
+ "path_exists": false,
493
+ "status": "pending",
494
+ "success": null,
495
+ "std_success": null,
496
+ "completed_seeds": null,
497
+ "num_completed": null,
498
+ "best_config": null,
499
+ "gain_vs_h16_policy": null
500
+ },
501
  {
502
  "key": "retrieval_residual_scale025_type_success025",
503
  "label": "Train-state residual retrieval, scale 0.25, train family success >= 0.25",
results/paper_table_status.md CHANGED
@@ -27,6 +27,8 @@ Baseline h=16 policy: 29.74%
27
  | retrieval_residual_scale050_safe_types | Train-state residual retrieval, scale 0.50, policy/no-op/wrong-gripper residuals | complete | 33.68% | +3.94 pp | yes | no | no | typed tangent-family mask ablation |
28
  | retrieval_residual_scale050_type_success010 | Train-state residual retrieval, scale 0.50, train family success >= 0.10 | complete | 33.33% | +3.59 pp | yes | no | no | train-split residual family reliability prior |
29
  | retrieval_residual_scale050_type_success025 | Train-state residual retrieval, scale 0.50, train family success >= 0.25 | complete | 33.33% | +3.59 pp | yes | no | no | train-split residual family reliability prior |
 
 
30
  | retrieval_residual_scale025_type_success025 | Train-state residual retrieval, scale 0.25, train family success >= 0.25 | complete | 32.93% | +3.19 pp | yes | no | no | train-split residual family reliability prior |
31
  | retrieval_residual_scale075 | Train-state residual retrieval, scale 0.75 | complete | 32.70% | +2.96 pp | yes | no | no | tangent transport scale ablation |
32
  | retrieval_residual_scale125 | Train-state residual retrieval, scale 1.25 | complete | 32.52% | +2.78 pp | yes | no | no | tangent transport scale ablation |
 
27
  | retrieval_residual_scale050_safe_types | Train-state residual retrieval, scale 0.50, policy/no-op/wrong-gripper residuals | complete | 33.68% | +3.94 pp | yes | no | no | typed tangent-family mask ablation |
28
  | retrieval_residual_scale050_type_success010 | Train-state residual retrieval, scale 0.50, train family success >= 0.10 | complete | 33.33% | +3.59 pp | yes | no | no | train-split residual family reliability prior |
29
  | retrieval_residual_scale050_type_success025 | Train-state residual retrieval, scale 0.50, train family success >= 0.25 | complete | 33.33% | +3.59 pp | yes | no | no | train-split residual family reliability prior |
30
+ | retrieval_residual_scale050_type_success050 | Train-state residual retrieval, scale 0.50, train family success >= 0.50 | pending 14859398/14859399 | pending | pending | yes | no | no | train-split residual family reliability prior |
31
+ | retrieval_residual_scale050_type_success075 | Train-state residual retrieval, scale 0.50, train family success >= 0.75 | pending 14859400/14859401 | pending | pending | yes | no | no | train-split residual family reliability prior |
32
  | retrieval_residual_scale025_type_success025 | Train-state residual retrieval, scale 0.25, train family success >= 0.25 | complete | 32.93% | +3.19 pp | yes | no | no | train-split residual family reliability prior |
33
  | retrieval_residual_scale075 | Train-state residual retrieval, scale 0.75 | complete | 32.70% | +2.96 pp | yes | no | no | tangent transport scale ablation |
34
  | retrieval_residual_scale125 | Train-state residual retrieval, scale 1.25 | complete | 32.52% | +2.78 pp | yes | no | no | tangent transport scale ablation |
scripts/build_paper_table_status.py CHANGED
@@ -255,6 +255,26 @@ SPECS = [
255
  story_role="train-split residual family reliability prior",
256
  pending_job="14859295/14859296",
257
  ),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
258
  ResultSpec(
259
  key="retrieval_residual_scale025_type_success025",
260
  label="Train-state residual retrieval, scale 0.25, train family success >= 0.25",
 
255
  story_role="train-split residual family reliability prior",
256
  pending_job="14859295/14859296",
257
  ),
258
+ ResultSpec(
259
+ key="retrieval_residual_scale050_type_success050",
260
+ label="Train-state residual retrieval, scale 0.50, train family success >= 0.50",
261
+ path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p50_type_success0p50_summary.json",
262
+ clean_deployment="yes",
263
+ same_state_proposals="no",
264
+ expert_proposal="no",
265
+ story_role="train-split residual family reliability prior",
266
+ pending_job="14859398/14859399",
267
+ ),
268
+ ResultSpec(
269
+ key="retrieval_residual_scale050_type_success075",
270
+ label="Train-state residual retrieval, scale 0.50, train family success >= 0.75",
271
+ path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p50_type_success0p75_summary.json",
272
+ clean_deployment="yes",
273
+ same_state_proposals="no",
274
+ expert_proposal="no",
275
+ story_role="train-split residual family reliability prior",
276
+ pending_job="14859400/14859401",
277
+ ),
278
  ResultSpec(
279
  key="retrieval_residual_scale025_type_success025",
280
  label="Train-state residual retrieval, scale 0.25, train family success >= 0.25",