anhtld commited on
Commit
8be4a18
·
verified ·
1 Parent(s): 258324d

Auto-sync: 2026-06-27 11:04:40

Browse files
results/h16_policy_ckpt_near_miss_policy_summary.json ADDED
@@ -0,0 +1,262 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
3
+ "objective": "near_miss_policy",
4
+ "out_name": "policy_rollout.json",
5
+ "num_completed": 3,
6
+ "baseline_h4_policy_success": 0.2967,
7
+ "baseline_h16_rank_checkpoint_success": 0.29739130434782607,
8
+ "mean_success": 0.2747826086956522,
9
+ "std_success": 0.03465192842638483,
10
+ "mean_progress": 0.5236737595651589,
11
+ "mean_action_mse_to_best": 0.3992425732257897,
12
+ "gain_vs_h4": -0.021917391304347833,
13
+ "gain_vs_h16_rank_checkpoint": -0.022608695652173882,
14
+ "rows": [
15
+ {
16
+ "seed": 0,
17
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy/seed_0/policy_rollout.json",
18
+ "num_groups": 575,
19
+ "selection_mode": "policy",
20
+ "num_candidates": 1,
21
+ "candidate_sigma": 0.0,
22
+ "policy_rollout_success_rate": 0.23478260869565218,
23
+ "policy_rollout_progress": 0.5030956517743028,
24
+ "oracle_success_rate": 0.8573913043478261,
25
+ "action_mse_to_best": 0.39332524563793253,
26
+ "best_policy_val": {
27
+ "bc_loss": 0.14066548272967339,
28
+ "field_effect_loss": 0.009273352101445198,
29
+ "field_potential_loss": 0.25917818148930866,
30
+ "field_preference_loss": 0.5168677667776743,
31
+ "lattice_edges": 3833.3333333333335,
32
+ "progress_mae": 0.19120572176244524,
33
+ "rank_acc": 0.827676342593299,
34
+ "rank_loss": 0.516864710383945,
35
+ "regret_mae": 0.37887536982695263,
36
+ "success_accuracy": 0.8731203774611155,
37
+ "total_loss": 1.0091058909893036
38
+ },
39
+ "per_task": {
40
+ "LiftPegUpright-v1": {
41
+ "action_mse_to_best": 0.3214439692782219,
42
+ "expert_success_rate": 0.8865979381443299,
43
+ "num_groups": 97,
44
+ "oracle_success_rate": 0.9278350515463918,
45
+ "policy_expert_regret": 1.0221107273372179,
46
+ "policy_oracle_regret": 1.0748680734757297,
47
+ "policy_rollout_progress": 0.6247779911326379,
48
+ "policy_rollout_success_rate": 0.21649484536082475,
49
+ "restore_max_error": 4.76837158203125e-07
50
+ },
51
+ "PickCube-v1": {
52
+ "action_mse_to_best": 0.30984279758726746,
53
+ "expert_success_rate": 0.9375,
54
+ "num_groups": 208,
55
+ "oracle_success_rate": 0.9471153846153846,
56
+ "policy_expert_regret": 1.4290453443088784,
57
+ "policy_oracle_regret": 1.4424682117353838,
58
+ "policy_rollout_progress": 0.45397288742690134,
59
+ "policy_rollout_success_rate": 0.028846153846153848,
60
+ "restore_max_error": 4.76837158203125e-07
61
+ },
62
+ "PullCube-v1": {
63
+ "action_mse_to_best": 0.6143633096249073,
64
+ "expert_success_rate": 0.19480519480519481,
65
+ "num_groups": 77,
66
+ "oracle_success_rate": 0.36363636363636365,
67
+ "policy_expert_regret": 0.2727886006090935,
68
+ "policy_oracle_regret": 0.4856883211956396,
69
+ "policy_rollout_progress": 0.28358456453719694,
70
+ "policy_rollout_success_rate": 0.16883116883116883,
71
+ "restore_max_error": 4.76837158203125e-07
72
+ },
73
+ "PushCube-v1": {
74
+ "action_mse_to_best": 0.3792672362178564,
75
+ "expert_success_rate": 0.8617021276595744,
76
+ "num_groups": 94,
77
+ "oracle_success_rate": 0.9893617021276596,
78
+ "policy_expert_regret": 0.4745611239303934,
79
+ "policy_oracle_regret": 0.5731581888617353,
80
+ "policy_rollout_progress": 0.7247141515637966,
81
+ "policy_rollout_success_rate": 0.7021276595744681,
82
+ "restore_max_error": 4.76837158203125e-07
83
+ },
84
+ "StackCube-v1": {
85
+ "action_mse_to_best": 0.48058135634419896,
86
+ "expert_success_rate": 0.6767676767676768,
87
+ "num_groups": 99,
88
+ "oracle_success_rate": 0.8585858585858586,
89
+ "policy_expert_regret": 0.9172844524034346,
90
+ "policy_oracle_regret": 1.104743935092531,
91
+ "policy_rollout_progress": 0.4473841649414313,
92
+ "policy_rollout_success_rate": 0.29292929292929293,
93
+ "restore_max_error": 3.948807716369629e-07
94
+ }
95
+ }
96
+ },
97
+ {
98
+ "seed": 1,
99
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy/seed_1/policy_rollout.json",
100
+ "num_groups": 575,
101
+ "selection_mode": "policy",
102
+ "num_candidates": 1,
103
+ "candidate_sigma": 0.0,
104
+ "policy_rollout_success_rate": 0.2956521739130435,
105
+ "policy_rollout_progress": 0.5316818758397384,
106
+ "oracle_success_rate": 0.8695652173913043,
107
+ "action_mse_to_best": 0.3977655249979833,
108
+ "best_policy_val": {
109
+ "bc_loss": 0.1323731690645218,
110
+ "field_effect_loss": 0.009979528995851675,
111
+ "field_potential_loss": 0.2860699908600913,
112
+ "field_preference_loss": 0.5113845500681136,
113
+ "lattice_edges": 3833.3333333333335,
114
+ "progress_mae": 0.19332215189933777,
115
+ "rank_acc": 0.8233457605044047,
116
+ "rank_loss": 0.5113737566603554,
117
+ "regret_mae": 0.35101599660184646,
118
+ "success_accuracy": 0.8738029234939151,
119
+ "total_loss": 1.0257672667503357
120
+ },
121
+ "per_task": {
122
+ "LiftPegUpright-v1": {
123
+ "action_mse_to_best": 0.32692277216377247,
124
+ "expert_success_rate": 0.8584070796460177,
125
+ "num_groups": 113,
126
+ "oracle_success_rate": 0.9380530973451328,
127
+ "policy_expert_regret": 1.0400315383366778,
128
+ "policy_oracle_regret": 1.1098291320083415,
129
+ "policy_rollout_progress": 0.6100680015783394,
130
+ "policy_rollout_success_rate": 0.21238938053097345,
131
+ "restore_max_error": 4.76837158203125e-07
132
+ },
133
+ "PickCube-v1": {
134
+ "action_mse_to_best": 0.323565029134245,
135
+ "expert_success_rate": 0.9402173913043478,
136
+ "num_groups": 184,
137
+ "oracle_success_rate": 0.9456521739130435,
138
+ "policy_expert_regret": 1.2229456127287173,
139
+ "policy_oracle_regret": 1.2229696172036473,
140
+ "policy_rollout_progress": 0.5365594722705124,
141
+ "policy_rollout_success_rate": 0.1793478260869565,
142
+ "restore_max_error": 4.76837158203125e-07
143
+ },
144
+ "PullCube-v1": {
145
+ "action_mse_to_best": 0.5897022130850115,
146
+ "expert_success_rate": 0.25,
147
+ "num_groups": 76,
148
+ "oracle_success_rate": 0.40789473684210525,
149
+ "policy_expert_regret": 0.25594949229248176,
150
+ "policy_oracle_regret": 0.510046625468883,
151
+ "policy_rollout_progress": 0.2688853383277953,
152
+ "policy_rollout_success_rate": 0.17105263157894737,
153
+ "restore_max_error": 4.76837158203125e-07
154
+ },
155
+ "PushCube-v1": {
156
+ "action_mse_to_best": 0.35923395164915034,
157
+ "expert_success_rate": 0.8198198198198198,
158
+ "num_groups": 111,
159
+ "oracle_success_rate": 1.0,
160
+ "policy_expert_regret": 0.3650765312953038,
161
+ "policy_oracle_regret": 0.48131698544497964,
162
+ "policy_rollout_progress": 0.7709352668072726,
163
+ "policy_rollout_success_rate": 0.7477477477477478,
164
+ "restore_max_error": 4.76837158203125e-07
165
+ },
166
+ "StackCube-v1": {
167
+ "action_mse_to_best": 0.5224681475946864,
168
+ "expert_success_rate": 0.6923076923076923,
169
+ "num_groups": 91,
170
+ "oracle_success_rate": 0.8571428571428571,
171
+ "policy_expert_regret": 1.1083575089226712,
172
+ "policy_oracle_regret": 1.280392975106344,
173
+ "policy_rollout_progress": 0.3521247384967385,
174
+ "policy_rollout_success_rate": 0.18681318681318682,
175
+ "restore_max_error": 3.948807716369629e-07
176
+ }
177
+ }
178
+ },
179
+ {
180
+ "seed": 2,
181
+ "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy/seed_2/policy_rollout.json",
182
+ "num_groups": 575,
183
+ "selection_mode": "policy",
184
+ "num_candidates": 1,
185
+ "candidate_sigma": 0.0,
186
+ "policy_rollout_success_rate": 0.29391304347826086,
187
+ "policy_rollout_progress": 0.5362437510814356,
188
+ "oracle_success_rate": 0.8765217391304347,
189
+ "action_mse_to_best": 0.40663694904145337,
190
+ "best_policy_val": {
191
+ "bc_loss": 0.11479559168219566,
192
+ "field_effect_loss": 0.009629687475454476,
193
+ "field_potential_loss": 0.26457422143883175,
194
+ "field_preference_loss": 0.5130132370524936,
195
+ "lattice_edges": 3833.3333333333335,
196
+ "progress_mae": 0.19729635367790857,
197
+ "rank_acc": 0.8342544337113699,
198
+ "rank_loss": 0.5129795521497726,
199
+ "regret_mae": 0.38137873510519665,
200
+ "success_accuracy": 0.8749299943447113,
201
+ "total_loss": 0.9850931962331136
202
+ },
203
+ "per_task": {
204
+ "LiftPegUpright-v1": {
205
+ "action_mse_to_best": 0.3428223121203094,
206
+ "expert_success_rate": 0.8229166666666666,
207
+ "num_groups": 96,
208
+ "oracle_success_rate": 0.9270833333333334,
209
+ "policy_expert_regret": 0.9818217330612242,
210
+ "policy_oracle_regret": 1.089017339516431,
211
+ "policy_rollout_progress": 0.5992394035371641,
212
+ "policy_rollout_success_rate": 0.21875,
213
+ "restore_max_error": 3.5762786865234375e-07
214
+ },
215
+ "PickCube-v1": {
216
+ "action_mse_to_best": 0.33260043210001905,
217
+ "expert_success_rate": 0.9444444444444444,
218
+ "num_groups": 198,
219
+ "oracle_success_rate": 0.9595959595959596,
220
+ "policy_expert_regret": 1.2200975062212709,
221
+ "policy_oracle_regret": 1.2413724587966848,
222
+ "policy_rollout_progress": 0.5395222330992722,
223
+ "policy_rollout_success_rate": 0.16161616161616163,
224
+ "restore_max_error": 4.76837158203125e-07
225
+ },
226
+ "PullCube-v1": {
227
+ "action_mse_to_best": 0.6126335874199867,
228
+ "expert_success_rate": 0.24444444444444444,
229
+ "num_groups": 90,
230
+ "oracle_success_rate": 0.4666666666666667,
231
+ "policy_expert_regret": 0.2349304822170072,
232
+ "policy_oracle_regret": 0.5191891225882702,
233
+ "policy_rollout_progress": 0.3588543026604586,
234
+ "policy_rollout_success_rate": 0.25555555555555554,
235
+ "restore_max_error": 4.0978193283081055e-07
236
+ },
237
+ "PushCube-v1": {
238
+ "action_mse_to_best": 0.3649870613866513,
239
+ "expert_success_rate": 0.8514851485148515,
240
+ "num_groups": 101,
241
+ "oracle_success_rate": 1.0,
242
+ "policy_expert_regret": 0.4751799466586349,
243
+ "policy_oracle_regret": 0.4948389716077559,
244
+ "policy_rollout_progress": 0.7625867709665015,
245
+ "policy_rollout_success_rate": 0.7425742574257426,
246
+ "restore_max_error": 4.76837158203125e-07
247
+ },
248
+ "StackCube-v1": {
249
+ "action_mse_to_best": 0.4783300234625737,
250
+ "expert_success_rate": 0.7666666666666667,
251
+ "num_groups": 90,
252
+ "oracle_success_rate": 0.9111111111111111,
253
+ "policy_expert_regret": 1.147929705844985,
254
+ "policy_oracle_regret": 1.3100251019001008,
255
+ "policy_rollout_progress": 0.3852180096838209,
256
+ "policy_rollout_success_rate": 0.2,
257
+ "restore_max_error": 4.76837158203125e-07
258
+ }
259
+ }
260
+ }
261
+ ]
262
+ }
results/h16_policy_ckpt_near_miss_policy_summary.md ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # h=16 Best-Policy Checkpoint Rollout
2
+
3
+ Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
4
+ Objective: `near_miss_policy`
5
+ Result file: `policy_rollout.json`
6
+ Completed seeds: 3
7
+ Baseline h=4 policy success: 29.67%
8
+ Baseline h=16 rank-checkpoint success: 29.74%
9
+
10
+ Mean success: 27.48% +/- 3.47%
11
+ Gain vs h=16 rank checkpoint: -2.26%
12
+ Mean progress: 52.37%
13
+ Mean action MSE to best: 0.399
14
+
15
+ | seed | mode | k | sigma | success | progress | oracle | action MSE |
16
+ |---:|---|---:|---:|---:|---:|---:|---:|
17
+ | 0 | policy | 1 | 0.00 | 23.48% | 50.31% | 85.74% | 0.393 |
18
+ | 1 | policy | 1 | 0.00 | 29.57% | 53.17% | 86.96% | 0.398 |
19
+ | 2 | policy | 1 | 0.00 | 29.39% | 53.62% | 87.65% | 0.407 |
results/h16_policy_ckpt_summary.json CHANGED
@@ -1,6 +1,7 @@
1
  {
2
  "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
3
  "objective": "base",
 
4
  "num_completed": 3,
5
  "baseline_h4_policy_success": 0.2967,
6
  "baseline_h16_rank_checkpoint_success": 0.29739130434782607,
@@ -15,6 +16,9 @@
15
  "seed": 0,
16
  "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/base/seed_0/policy_rollout.json",
17
  "num_groups": 575,
 
 
 
18
  "policy_rollout_success_rate": 0.2747826086956522,
19
  "policy_rollout_progress": 0.530730156995022,
20
  "oracle_success_rate": 0.8573913043478261,
@@ -94,6 +98,9 @@
94
  "seed": 1,
95
  "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/base/seed_1/policy_rollout.json",
96
  "num_groups": 575,
 
 
 
97
  "policy_rollout_success_rate": 0.23304347826086957,
98
  "policy_rollout_progress": 0.4766378657271464,
99
  "oracle_success_rate": 0.8695652173913043,
@@ -173,6 +180,9 @@
173
  "seed": 2,
174
  "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/base/seed_2/policy_rollout.json",
175
  "num_groups": 575,
 
 
 
176
  "policy_rollout_success_rate": 0.3026086956521739,
177
  "policy_rollout_progress": 0.5308780688630498,
178
  "oracle_success_rate": 0.8765217391304347,
 
1
  {
2
  "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
3
  "objective": "base",
4
+ "out_name": "policy_rollout.json",
5
  "num_completed": 3,
6
  "baseline_h4_policy_success": 0.2967,
7
  "baseline_h16_rank_checkpoint_success": 0.29739130434782607,
 
16
  "seed": 0,
17
  "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/base/seed_0/policy_rollout.json",
18
  "num_groups": 575,
19
+ "selection_mode": "policy",
20
+ "num_candidates": 1,
21
+ "candidate_sigma": 0.0,
22
  "policy_rollout_success_rate": 0.2747826086956522,
23
  "policy_rollout_progress": 0.530730156995022,
24
  "oracle_success_rate": 0.8573913043478261,
 
98
  "seed": 1,
99
  "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/base/seed_1/policy_rollout.json",
100
  "num_groups": 575,
101
+ "selection_mode": "policy",
102
+ "num_candidates": 1,
103
+ "candidate_sigma": 0.0,
104
  "policy_rollout_success_rate": 0.23304347826086957,
105
  "policy_rollout_progress": 0.4766378657271464,
106
  "oracle_success_rate": 0.8695652173913043,
 
180
  "seed": 2,
181
  "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/base/seed_2/policy_rollout.json",
182
  "num_groups": 575,
183
+ "selection_mode": "policy",
184
+ "num_candidates": 1,
185
+ "candidate_sigma": 0.0,
186
  "policy_rollout_success_rate": 0.3026086956521739,
187
  "policy_rollout_progress": 0.5308780688630498,
188
  "oracle_success_rate": 0.8765217391304347,
results/h16_policy_ckpt_summary.md CHANGED
@@ -1,6 +1,8 @@
1
  # h=16 Best-Policy Checkpoint Rollout
2
 
3
  Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
 
 
4
  Completed seeds: 3
5
  Baseline h=4 policy success: 29.67%
6
  Baseline h=16 rank-checkpoint success: 29.74%
@@ -10,8 +12,8 @@ Gain vs h=16 rank checkpoint: -2.72%
10
  Mean progress: 51.27%
11
  Mean action MSE to best: 0.411
12
 
13
- | seed | success | progress | oracle | action MSE |
14
- |---:|---:|---:|---:|---:|
15
- | 0 | 27.48% | 53.07% | 85.74% | 0.384 |
16
- | 1 | 23.30% | 47.66% | 86.96% | 0.423 |
17
- | 2 | 30.26% | 53.09% | 87.65% | 0.427 |
 
1
  # h=16 Best-Policy Checkpoint Rollout
2
 
3
  Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
4
+ Objective: `base`
5
+ Result file: `policy_rollout.json`
6
  Completed seeds: 3
7
  Baseline h=4 policy success: 29.67%
8
  Baseline h=16 rank-checkpoint success: 29.74%
 
12
  Mean progress: 51.27%
13
  Mean action MSE to best: 0.411
14
 
15
+ | seed | mode | k | sigma | success | progress | oracle | action MSE |
16
+ |---:|---|---:|---:|---:|---:|---:|---:|
17
+ | 0 | policy | 1 | 0.00 | 27.48% | 53.07% | 85.74% | 0.384 |
18
+ | 1 | policy | 1 | 0.00 | 23.30% | 47.66% | 86.96% | 0.423 |
19
+ | 2 | policy | 1 | 0.00 | 30.26% | 53.09% | 87.65% | 0.427 |