Auto-sync: 2026-06-28 08:39:56
Browse files- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p35_safe_types_margin0p15_summary.json +292 -0
- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p35_safe_types_margin0p15_summary.md +19 -0
- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p35_safe_types_margin0p20_summary.json +292 -0
- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p35_safe_types_margin0p20_summary.md +19 -0
- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p35_safe_types_margin0p25_summary.json +292 -0
- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p35_safe_types_margin0p25_summary.md +19 -0
- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p40_safe_types_margin0p20_summary.json +292 -0
- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p40_safe_types_margin0p20_summary.md +19 -0
- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p50_safe_types_margin0p20_summary.json +292 -0
- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p50_safe_types_margin0p20_summary.md +19 -0
- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn4_scale0p35_safe_types_margin0p20_summary.json +292 -0
- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn4_scale0p35_safe_types_margin0p20_summary.md +19 -0
- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn8_scale0p35_safe_types_margin0p20_summary.json +292 -0
- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn8_scale0p35_safe_types_margin0p20_summary.md +19 -0
- results/paper_table_status.json +28 -9
- results/paper_table_status.md +2 -1
- scripts/build_paper_table_status.py +10 -0
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p35_safe_types_margin0p15_summary.json
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
|
| 3 |
+
"objective": "near_miss_policy_bc5",
|
| 4 |
+
"out_name": "policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p15.json",
|
| 5 |
+
"num_completed": 3,
|
| 6 |
+
"baseline_h4_policy_success": 0.2967,
|
| 7 |
+
"baseline_h16_rank_checkpoint_success": 0.29739130434782607,
|
| 8 |
+
"mean_success": 0.3472463768115942,
|
| 9 |
+
"std_success": 0.012581179370556939,
|
| 10 |
+
"mean_progress": 0.564756924334041,
|
| 11 |
+
"mean_action_mse_to_best": 0.4001109910572785,
|
| 12 |
+
"gain_vs_h4": 0.050546376811594185,
|
| 13 |
+
"gain_vs_h16_rank_checkpoint": 0.049855072463768135,
|
| 14 |
+
"rows": [
|
| 15 |
+
{
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p15.json",
|
| 18 |
+
"num_groups": 575,
|
| 19 |
+
"selection_mode": "retrieval_residual",
|
| 20 |
+
"num_candidates": 32,
|
| 21 |
+
"candidate_sigma": 0.0,
|
| 22 |
+
"selection_margin": 0.15,
|
| 23 |
+
"field_optim_steps": 0,
|
| 24 |
+
"field_optim_step_size": 0.0,
|
| 25 |
+
"field_optim_trust_radius": 0.0,
|
| 26 |
+
"field_optim_l2_penalty": 0.0,
|
| 27 |
+
"retrieval_neighbors": 2,
|
| 28 |
+
"retrieval_metric": "raw",
|
| 29 |
+
"retrieval_type_min_success": 0.0,
|
| 30 |
+
"retrieval_residual_scale": 0.35,
|
| 31 |
+
"retrieval_residual_anchor": "expert",
|
| 32 |
+
"policy_rollout_success_rate": 0.3408695652173913,
|
| 33 |
+
"policy_rollout_progress": 0.5531754332145109,
|
| 34 |
+
"oracle_success_rate": 0.8573913043478261,
|
| 35 |
+
"action_mse_to_best": 0.3833285608312682,
|
| 36 |
+
"best_policy_val": {
|
| 37 |
+
"bc_loss": 0.13721593966086706,
|
| 38 |
+
"field_effect_loss": 0.009290305380192068,
|
| 39 |
+
"field_potential_loss": 0.2666468388504452,
|
| 40 |
+
"field_preference_loss": 0.5130573478009965,
|
| 41 |
+
"lattice_edges": 3833.3333333333335,
|
| 42 |
+
"progress_mae": 0.1933159919248687,
|
| 43 |
+
"rank_acc": 0.8265031774838766,
|
| 44 |
+
"rank_loss": 0.5130523675017886,
|
| 45 |
+
"regret_mae": 0.3756548762321472,
|
| 46 |
+
"success_accuracy": 0.8773836526605818,
|
| 47 |
+
"total_loss": 1.5581054819954767
|
| 48 |
+
},
|
| 49 |
+
"per_task": {
|
| 50 |
+
"LiftPegUpright-v1": {
|
| 51 |
+
"action_mse_to_best": 0.31969772514973566,
|
| 52 |
+
"expert_success_rate": 0.8865979381443299,
|
| 53 |
+
"num_groups": 97,
|
| 54 |
+
"oracle_success_rate": 0.9278350515463918,
|
| 55 |
+
"policy_expert_regret": 1.0792337135248578,
|
| 56 |
+
"policy_oracle_regret": 1.1056491912333006,
|
| 57 |
+
"policy_rollout_progress": 0.570563180084081,
|
| 58 |
+
"policy_rollout_success_rate": 0.2268041237113402,
|
| 59 |
+
"restore_max_error": 4.76837158203125e-07
|
| 60 |
+
},
|
| 61 |
+
"PickCube-v1": {
|
| 62 |
+
"action_mse_to_best": 0.3030671978440995,
|
| 63 |
+
"expert_success_rate": 0.9375,
|
| 64 |
+
"num_groups": 208,
|
| 65 |
+
"oracle_success_rate": 0.9471153846153846,
|
| 66 |
+
"policy_expert_regret": 0.9685039922523384,
|
| 67 |
+
"policy_oracle_regret": 0.9819268596788439,
|
| 68 |
+
"policy_rollout_progress": 0.613570343882132,
|
| 69 |
+
"policy_rollout_success_rate": 0.33653846153846156,
|
| 70 |
+
"restore_max_error": 4.76837158203125e-07
|
| 71 |
+
},
|
| 72 |
+
"PullCube-v1": {
|
| 73 |
+
"action_mse_to_best": 0.6005318368023093,
|
| 74 |
+
"expert_success_rate": 0.19480519480519481,
|
| 75 |
+
"num_groups": 77,
|
| 76 |
+
"oracle_success_rate": 0.36363636363636365,
|
| 77 |
+
"policy_expert_regret": 0.19676480796974297,
|
| 78 |
+
"policy_oracle_regret": 0.3678719857979759,
|
| 79 |
+
"policy_rollout_progress": 0.34563462398402006,
|
| 80 |
+
"policy_rollout_success_rate": 0.24675324675324675,
|
| 81 |
+
"restore_max_error": 4.76837158203125e-07
|
| 82 |
+
},
|
| 83 |
+
"PushCube-v1": {
|
| 84 |
+
"action_mse_to_best": 0.36560009463511883,
|
| 85 |
+
"expert_success_rate": 0.8617021276595744,
|
| 86 |
+
"num_groups": 94,
|
| 87 |
+
"oracle_success_rate": 0.9893617021276596,
|
| 88 |
+
"policy_expert_regret": 0.5719942746961371,
|
| 89 |
+
"policy_oracle_regret": 0.6683584781403237,
|
| 90 |
+
"policy_rollout_progress": 0.6827053516469104,
|
| 91 |
+
"policy_rollout_success_rate": 0.648936170212766,
|
| 92 |
+
"restore_max_error": 4.76837158203125e-07
|
| 93 |
+
},
|
| 94 |
+
"StackCube-v1": {
|
| 95 |
+
"action_mse_to_best": 0.4622010672464967,
|
| 96 |
+
"expert_success_rate": 0.6767676767676768,
|
| 97 |
+
"num_groups": 99,
|
| 98 |
+
"oracle_success_rate": 0.8585858585858586,
|
| 99 |
+
"policy_expert_regret": 0.9681115684786228,
|
| 100 |
+
"policy_oracle_regret": 1.1308042419077171,
|
| 101 |
+
"policy_rollout_progress": 0.44768126263762964,
|
| 102 |
+
"policy_rollout_success_rate": 0.24242424242424243,
|
| 103 |
+
"restore_max_error": 3.948807716369629e-07
|
| 104 |
+
}
|
| 105 |
+
}
|
| 106 |
+
},
|
| 107 |
+
{
|
| 108 |
+
"seed": 1,
|
| 109 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p15.json",
|
| 110 |
+
"num_groups": 575,
|
| 111 |
+
"selection_mode": "retrieval_residual",
|
| 112 |
+
"num_candidates": 32,
|
| 113 |
+
"candidate_sigma": 0.0,
|
| 114 |
+
"selection_margin": 0.15,
|
| 115 |
+
"field_optim_steps": 0,
|
| 116 |
+
"field_optim_step_size": 0.0,
|
| 117 |
+
"field_optim_trust_radius": 0.0,
|
| 118 |
+
"field_optim_l2_penalty": 0.0,
|
| 119 |
+
"retrieval_neighbors": 2,
|
| 120 |
+
"retrieval_metric": "raw",
|
| 121 |
+
"retrieval_type_min_success": 0.0,
|
| 122 |
+
"retrieval_residual_scale": 0.35,
|
| 123 |
+
"retrieval_residual_anchor": "expert",
|
| 124 |
+
"policy_rollout_success_rate": 0.3391304347826087,
|
| 125 |
+
"policy_rollout_progress": 0.5647912328366352,
|
| 126 |
+
"oracle_success_rate": 0.8695652173913043,
|
| 127 |
+
"action_mse_to_best": 0.39385083786495356,
|
| 128 |
+
"best_policy_val": {
|
| 129 |
+
"bc_loss": 0.1315989200439718,
|
| 130 |
+
"field_effect_loss": 0.010110370814800262,
|
| 131 |
+
"field_potential_loss": 0.31546536915832096,
|
| 132 |
+
"field_preference_loss": 0.4960637440284093,
|
| 133 |
+
"lattice_edges": 3833.3333333333335,
|
| 134 |
+
"progress_mae": 0.19023226698239645,
|
| 135 |
+
"rank_acc": 0.8255469501018524,
|
| 136 |
+
"rank_loss": 0.4960531195004781,
|
| 137 |
+
"regret_mae": 0.42156532737943864,
|
| 138 |
+
"success_accuracy": 0.8807473679383596,
|
| 139 |
+
"total_loss": 1.561984618504842
|
| 140 |
+
},
|
| 141 |
+
"per_task": {
|
| 142 |
+
"LiftPegUpright-v1": {
|
| 143 |
+
"action_mse_to_best": 0.32597753190460194,
|
| 144 |
+
"expert_success_rate": 0.8584070796460177,
|
| 145 |
+
"num_groups": 113,
|
| 146 |
+
"oracle_success_rate": 0.9380530973451328,
|
| 147 |
+
"policy_expert_regret": 0.987694250535121,
|
| 148 |
+
"policy_oracle_regret": 1.066896405631462,
|
| 149 |
+
"policy_rollout_progress": 0.6352153201546289,
|
| 150 |
+
"policy_rollout_success_rate": 0.23008849557522124,
|
| 151 |
+
"restore_max_error": 4.76837158203125e-07
|
| 152 |
+
},
|
| 153 |
+
"PickCube-v1": {
|
| 154 |
+
"action_mse_to_best": 0.3230286493452023,
|
| 155 |
+
"expert_success_rate": 0.9402173913043478,
|
| 156 |
+
"num_groups": 184,
|
| 157 |
+
"oracle_success_rate": 0.9456521739130435,
|
| 158 |
+
"policy_expert_regret": 1.0767439803027588,
|
| 159 |
+
"policy_oracle_regret": 1.076767984777689,
|
| 160 |
+
"policy_rollout_progress": 0.5738926515307116,
|
| 161 |
+
"policy_rollout_success_rate": 0.28804347826086957,
|
| 162 |
+
"restore_max_error": 4.76837158203125e-07
|
| 163 |
+
},
|
| 164 |
+
"PullCube-v1": {
|
| 165 |
+
"action_mse_to_best": 0.5854560756369641,
|
| 166 |
+
"expert_success_rate": 0.25,
|
| 167 |
+
"num_groups": 76,
|
| 168 |
+
"oracle_success_rate": 0.40789473684210525,
|
| 169 |
+
"policy_expert_regret": 0.2998436244547759,
|
| 170 |
+
"policy_oracle_regret": 0.547350984679437,
|
| 171 |
+
"policy_rollout_progress": 0.2787476073901512,
|
| 172 |
+
"policy_rollout_success_rate": 0.17105263157894737,
|
| 173 |
+
"restore_max_error": 4.76837158203125e-07
|
| 174 |
+
},
|
| 175 |
+
"PushCube-v1": {
|
| 176 |
+
"action_mse_to_best": 0.3573181240206903,
|
| 177 |
+
"expert_success_rate": 0.8198198198198198,
|
| 178 |
+
"num_groups": 111,
|
| 179 |
+
"oracle_success_rate": 1.0,
|
| 180 |
+
"policy_expert_regret": 0.3476516243305292,
|
| 181 |
+
"policy_oracle_regret": 0.3811321313585247,
|
| 182 |
+
"policy_rollout_progress": 0.8170660668396735,
|
| 183 |
+
"policy_rollout_success_rate": 0.8018018018018018,
|
| 184 |
+
"restore_max_error": 4.76837158203125e-07
|
| 185 |
+
},
|
| 186 |
+
"StackCube-v1": {
|
| 187 |
+
"action_mse_to_best": 0.5058739084934631,
|
| 188 |
+
"expert_success_rate": 0.6923076923076923,
|
| 189 |
+
"num_groups": 91,
|
| 190 |
+
"oracle_success_rate": 0.8571428571428571,
|
| 191 |
+
"policy_expert_regret": 1.128345174776329,
|
| 192 |
+
"policy_oracle_regret": 1.3004929289058014,
|
| 193 |
+
"policy_rollout_progress": 0.39011239825369237,
|
| 194 |
+
"policy_rollout_success_rate": 0.15384615384615385,
|
| 195 |
+
"restore_max_error": 3.948807716369629e-07
|
| 196 |
+
}
|
| 197 |
+
}
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"seed": 2,
|
| 201 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p15.json",
|
| 202 |
+
"num_groups": 575,
|
| 203 |
+
"selection_mode": "retrieval_residual",
|
| 204 |
+
"num_candidates": 32,
|
| 205 |
+
"candidate_sigma": 0.0,
|
| 206 |
+
"selection_margin": 0.15,
|
| 207 |
+
"field_optim_steps": 0,
|
| 208 |
+
"field_optim_step_size": 0.0,
|
| 209 |
+
"field_optim_trust_radius": 0.0,
|
| 210 |
+
"field_optim_l2_penalty": 0.0,
|
| 211 |
+
"retrieval_neighbors": 2,
|
| 212 |
+
"retrieval_metric": "raw",
|
| 213 |
+
"retrieval_type_min_success": 0.0,
|
| 214 |
+
"retrieval_residual_scale": 0.35,
|
| 215 |
+
"retrieval_residual_anchor": "expert",
|
| 216 |
+
"policy_rollout_success_rate": 0.3617391304347826,
|
| 217 |
+
"policy_rollout_progress": 0.5763041069509769,
|
| 218 |
+
"oracle_success_rate": 0.8765217391304347,
|
| 219 |
+
"action_mse_to_best": 0.42315357447561364,
|
| 220 |
+
"best_policy_val": {
|
| 221 |
+
"bc_loss": 0.11367896075050037,
|
| 222 |
+
"field_effect_loss": 0.009670218582161598,
|
| 223 |
+
"field_potential_loss": 0.2641640139950646,
|
| 224 |
+
"field_preference_loss": 0.5130490180518892,
|
| 225 |
+
"lattice_edges": 3833.3333333333335,
|
| 226 |
+
"progress_mae": 0.2021729110015763,
|
| 227 |
+
"rank_acc": 0.8333857821093665,
|
| 228 |
+
"rank_loss": 0.5130119257503085,
|
| 229 |
+
"regret_mae": 0.3958987047274907,
|
| 230 |
+
"success_accuracy": 0.8680730561415354,
|
| 231 |
+
"total_loss": 1.4394984311527677
|
| 232 |
+
},
|
| 233 |
+
"per_task": {
|
| 234 |
+
"LiftPegUpright-v1": {
|
| 235 |
+
"action_mse_to_best": 0.35563401411733747,
|
| 236 |
+
"expert_success_rate": 0.8229166666666666,
|
| 237 |
+
"num_groups": 96,
|
| 238 |
+
"oracle_success_rate": 0.9270833333333334,
|
| 239 |
+
"policy_expert_regret": 0.8605156441529592,
|
| 240 |
+
"policy_oracle_regret": 0.966790359467268,
|
| 241 |
+
"policy_rollout_progress": 0.6439014971256256,
|
| 242 |
+
"policy_rollout_success_rate": 0.3229166666666667,
|
| 243 |
+
"restore_max_error": 3.5762786865234375e-07
|
| 244 |
+
},
|
| 245 |
+
"PickCube-v1": {
|
| 246 |
+
"action_mse_to_best": 0.337729223580523,
|
| 247 |
+
"expert_success_rate": 0.9444444444444444,
|
| 248 |
+
"num_groups": 198,
|
| 249 |
+
"oracle_success_rate": 0.9595959595959596,
|
| 250 |
+
"policy_expert_regret": 0.9788274636510947,
|
| 251 |
+
"policy_oracle_regret": 0.9859109141798031,
|
| 252 |
+
"policy_rollout_progress": 0.6305044059914471,
|
| 253 |
+
"policy_rollout_success_rate": 0.3383838383838384,
|
| 254 |
+
"restore_max_error": 4.76837158203125e-07
|
| 255 |
+
},
|
| 256 |
+
"PullCube-v1": {
|
| 257 |
+
"action_mse_to_best": 0.6612004579769241,
|
| 258 |
+
"expert_success_rate": 0.24444444444444444,
|
| 259 |
+
"num_groups": 90,
|
| 260 |
+
"oracle_success_rate": 0.4666666666666667,
|
| 261 |
+
"policy_expert_regret": 0.31387021076623367,
|
| 262 |
+
"policy_oracle_regret": 0.5408862758082856,
|
| 263 |
+
"policy_rollout_progress": 0.3207382236021052,
|
| 264 |
+
"policy_rollout_success_rate": 0.2111111111111111,
|
| 265 |
+
"restore_max_error": 4.0978193283081055e-07
|
| 266 |
+
},
|
| 267 |
+
"PushCube-v1": {
|
| 268 |
+
"action_mse_to_best": 0.38559474906708935,
|
| 269 |
+
"expert_success_rate": 0.8514851485148515,
|
| 270 |
+
"num_groups": 101,
|
| 271 |
+
"oracle_success_rate": 1.0,
|
| 272 |
+
"policy_expert_regret": 0.3989662665246737,
|
| 273 |
+
"policy_oracle_regret": 0.4538695028807857,
|
| 274 |
+
"policy_rollout_progress": 0.7837542594954519,
|
| 275 |
+
"policy_rollout_success_rate": 0.7623762376237624,
|
| 276 |
+
"restore_max_error": 4.76837158203125e-07
|
| 277 |
+
},
|
| 278 |
+
"StackCube-v1": {
|
| 279 |
+
"action_mse_to_best": 0.4872104758396745,
|
| 280 |
+
"expert_success_rate": 0.7666666666666667,
|
| 281 |
+
"num_groups": 90,
|
| 282 |
+
"oracle_success_rate": 0.9111111111111111,
|
| 283 |
+
"policy_expert_regret": 1.169666932688819,
|
| 284 |
+
"policy_oracle_regret": 1.3187390978137652,
|
| 285 |
+
"policy_rollout_progress": 0.40772027836905583,
|
| 286 |
+
"policy_rollout_success_rate": 0.15555555555555556,
|
| 287 |
+
"restore_max_error": 4.76837158203125e-07
|
| 288 |
+
}
|
| 289 |
+
}
|
| 290 |
+
}
|
| 291 |
+
]
|
| 292 |
+
}
|
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p35_safe_types_margin0p15_summary.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# h=16 Best-Policy Checkpoint Rollout
|
| 2 |
+
|
| 3 |
+
Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
|
| 4 |
+
Objective: `near_miss_policy_bc5`
|
| 5 |
+
Result file: `policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p15.json`
|
| 6 |
+
Completed seeds: 3
|
| 7 |
+
Baseline h=4 policy success: 29.67%
|
| 8 |
+
Baseline h=16 rank-checkpoint success: 29.74%
|
| 9 |
+
|
| 10 |
+
Mean success: 34.72% +/- 1.26%
|
| 11 |
+
Gain vs h=16 rank checkpoint: +4.99%
|
| 12 |
+
Mean progress: 56.48%
|
| 13 |
+
Mean action MSE to best: 0.400
|
| 14 |
+
|
| 15 |
+
| seed | mode | k | retrieval K | retrieval metric | residual anchor | min type success | residual scale | margin | sigma | opt steps | trust | success | progress | oracle | action MSE |
|
| 16 |
+
|---:|---|---:|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
| 17 |
+
| 0 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.35 | 0.150 | 0.00 | 0 | 0.00 | 34.09% | 55.32% | 85.74% | 0.383 |
|
| 18 |
+
| 1 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.35 | 0.150 | 0.00 | 0 | 0.00 | 33.91% | 56.48% | 86.96% | 0.394 |
|
| 19 |
+
| 2 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.35 | 0.150 | 0.00 | 0 | 0.00 | 36.17% | 57.63% | 87.65% | 0.423 |
|
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p35_safe_types_margin0p20_summary.json
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
|
| 3 |
+
"objective": "near_miss_policy_bc5",
|
| 4 |
+
"out_name": "policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p20.json",
|
| 5 |
+
"num_completed": 3,
|
| 6 |
+
"baseline_h4_policy_success": 0.2967,
|
| 7 |
+
"baseline_h16_rank_checkpoint_success": 0.29739130434782607,
|
| 8 |
+
"mean_success": 0.3501449275362319,
|
| 9 |
+
"std_success": 0.01907766106887459,
|
| 10 |
+
"mean_progress": 0.5666251124668834,
|
| 11 |
+
"mean_action_mse_to_best": 0.39739443172795186,
|
| 12 |
+
"gain_vs_h4": 0.05344492753623187,
|
| 13 |
+
"gain_vs_h16_rank_checkpoint": 0.05275362318840582,
|
| 14 |
+
"rows": [
|
| 15 |
+
{
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p20.json",
|
| 18 |
+
"num_groups": 575,
|
| 19 |
+
"selection_mode": "retrieval_residual",
|
| 20 |
+
"num_candidates": 32,
|
| 21 |
+
"candidate_sigma": 0.0,
|
| 22 |
+
"selection_margin": 0.2,
|
| 23 |
+
"field_optim_steps": 0,
|
| 24 |
+
"field_optim_step_size": 0.0,
|
| 25 |
+
"field_optim_trust_radius": 0.0,
|
| 26 |
+
"field_optim_l2_penalty": 0.0,
|
| 27 |
+
"retrieval_neighbors": 2,
|
| 28 |
+
"retrieval_metric": "raw",
|
| 29 |
+
"retrieval_type_min_success": 0.0,
|
| 30 |
+
"retrieval_residual_scale": 0.35,
|
| 31 |
+
"retrieval_residual_anchor": "expert",
|
| 32 |
+
"policy_rollout_success_rate": 0.3391304347826087,
|
| 33 |
+
"policy_rollout_progress": 0.5522897216171512,
|
| 34 |
+
"oracle_success_rate": 0.8573913043478261,
|
| 35 |
+
"action_mse_to_best": 0.38128977080969056,
|
| 36 |
+
"best_policy_val": {
|
| 37 |
+
"bc_loss": 0.13721593966086706,
|
| 38 |
+
"field_effect_loss": 0.009290305380192068,
|
| 39 |
+
"field_potential_loss": 0.2666468388504452,
|
| 40 |
+
"field_preference_loss": 0.5130573478009965,
|
| 41 |
+
"lattice_edges": 3833.3333333333335,
|
| 42 |
+
"progress_mae": 0.1933159919248687,
|
| 43 |
+
"rank_acc": 0.8265031774838766,
|
| 44 |
+
"rank_loss": 0.5130523675017886,
|
| 45 |
+
"regret_mae": 0.3756548762321472,
|
| 46 |
+
"success_accuracy": 0.8773836526605818,
|
| 47 |
+
"total_loss": 1.5581054819954767
|
| 48 |
+
},
|
| 49 |
+
"per_task": {
|
| 50 |
+
"LiftPegUpright-v1": {
|
| 51 |
+
"action_mse_to_best": 0.31969772514973566,
|
| 52 |
+
"expert_success_rate": 0.8865979381443299,
|
| 53 |
+
"num_groups": 97,
|
| 54 |
+
"oracle_success_rate": 0.9278350515463918,
|
| 55 |
+
"policy_expert_regret": 1.0792337135248578,
|
| 56 |
+
"policy_oracle_regret": 1.1056491912333006,
|
| 57 |
+
"policy_rollout_progress": 0.570563180084081,
|
| 58 |
+
"policy_rollout_success_rate": 0.2268041237113402,
|
| 59 |
+
"restore_max_error": 4.76837158203125e-07
|
| 60 |
+
},
|
| 61 |
+
"PickCube-v1": {
|
| 62 |
+
"action_mse_to_best": 0.30274841501019323,
|
| 63 |
+
"expert_success_rate": 0.9375,
|
| 64 |
+
"num_groups": 208,
|
| 65 |
+
"oracle_success_rate": 0.9471153846153846,
|
| 66 |
+
"policy_expert_regret": 0.9682457399411271,
|
| 67 |
+
"policy_oracle_regret": 0.9816686073676325,
|
| 68 |
+
"policy_rollout_progress": 0.6138285961933434,
|
| 69 |
+
"policy_rollout_success_rate": 0.33653846153846156,
|
| 70 |
+
"restore_max_error": 4.76837158203125e-07
|
| 71 |
+
},
|
| 72 |
+
"PullCube-v1": {
|
| 73 |
+
"action_mse_to_best": 0.6005318368023093,
|
| 74 |
+
"expert_success_rate": 0.19480519480519481,
|
| 75 |
+
"num_groups": 77,
|
| 76 |
+
"oracle_success_rate": 0.36363636363636365,
|
| 77 |
+
"policy_expert_regret": 0.19676480796974297,
|
| 78 |
+
"policy_oracle_regret": 0.3678719857979759,
|
| 79 |
+
"policy_rollout_progress": 0.34563462398402006,
|
| 80 |
+
"policy_rollout_success_rate": 0.24675324675324675,
|
| 81 |
+
"restore_max_error": 4.76837158203125e-07
|
| 82 |
+
},
|
| 83 |
+
"PushCube-v1": {
|
| 84 |
+
"action_mse_to_best": 0.36560009463511883,
|
| 85 |
+
"expert_success_rate": 0.8617021276595744,
|
| 86 |
+
"num_groups": 94,
|
| 87 |
+
"oracle_success_rate": 0.9893617021276596,
|
| 88 |
+
"policy_expert_regret": 0.5719942746961371,
|
| 89 |
+
"policy_oracle_regret": 0.6683584781403237,
|
| 90 |
+
"policy_rollout_progress": 0.6827053516469104,
|
| 91 |
+
"policy_rollout_success_rate": 0.648936170212766,
|
| 92 |
+
"restore_max_error": 4.76837158203125e-07
|
| 93 |
+
},
|
| 94 |
+
"StackCube-v1": {
|
| 95 |
+
"action_mse_to_best": 0.4510293760045309,
|
| 96 |
+
"expert_success_rate": 0.6767676767676768,
|
| 97 |
+
"num_groups": 99,
|
| 98 |
+
"oracle_success_rate": 0.8585858585858586,
|
| 99 |
+
"policy_expert_regret": 0.9724158290961776,
|
| 100 |
+
"policy_oracle_regret": 1.1465921272533108,
|
| 101 |
+
"policy_rollout_progress": 0.4419943873930459,
|
| 102 |
+
"policy_rollout_success_rate": 0.23232323232323232,
|
| 103 |
+
"restore_max_error": 3.948807716369629e-07
|
| 104 |
+
}
|
| 105 |
+
}
|
| 106 |
+
},
|
| 107 |
+
{
|
| 108 |
+
"seed": 1,
|
| 109 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p20.json",
|
| 110 |
+
"num_groups": 575,
|
| 111 |
+
"selection_mode": "retrieval_residual",
|
| 112 |
+
"num_candidates": 32,
|
| 113 |
+
"candidate_sigma": 0.0,
|
| 114 |
+
"selection_margin": 0.2,
|
| 115 |
+
"field_optim_steps": 0,
|
| 116 |
+
"field_optim_step_size": 0.0,
|
| 117 |
+
"field_optim_trust_radius": 0.0,
|
| 118 |
+
"field_optim_l2_penalty": 0.0,
|
| 119 |
+
"retrieval_neighbors": 2,
|
| 120 |
+
"retrieval_metric": "raw",
|
| 121 |
+
"retrieval_type_min_success": 0.0,
|
| 122 |
+
"retrieval_residual_scale": 0.35,
|
| 123 |
+
"retrieval_residual_anchor": "expert",
|
| 124 |
+
"policy_rollout_success_rate": 0.3391304347826087,
|
| 125 |
+
"policy_rollout_progress": 0.5639680514419856,
|
| 126 |
+
"oracle_success_rate": 0.8695652173913043,
|
| 127 |
+
"action_mse_to_best": 0.3912962566478097,
|
| 128 |
+
"best_policy_val": {
|
| 129 |
+
"bc_loss": 0.1315989200439718,
|
| 130 |
+
"field_effect_loss": 0.010110370814800262,
|
| 131 |
+
"field_potential_loss": 0.31546536915832096,
|
| 132 |
+
"field_preference_loss": 0.4960637440284093,
|
| 133 |
+
"lattice_edges": 3833.3333333333335,
|
| 134 |
+
"progress_mae": 0.19023226698239645,
|
| 135 |
+
"rank_acc": 0.8255469501018524,
|
| 136 |
+
"rank_loss": 0.4960531195004781,
|
| 137 |
+
"regret_mae": 0.42156532737943864,
|
| 138 |
+
"success_accuracy": 0.8807473679383596,
|
| 139 |
+
"total_loss": 1.561984618504842
|
| 140 |
+
},
|
| 141 |
+
"per_task": {
|
| 142 |
+
"LiftPegUpright-v1": {
|
| 143 |
+
"action_mse_to_best": 0.32597753190460194,
|
| 144 |
+
"expert_success_rate": 0.8584070796460177,
|
| 145 |
+
"num_groups": 113,
|
| 146 |
+
"oracle_success_rate": 0.9380530973451328,
|
| 147 |
+
"policy_expert_regret": 0.987694250535121,
|
| 148 |
+
"policy_oracle_regret": 1.066896405631462,
|
| 149 |
+
"policy_rollout_progress": 0.6352153201546289,
|
| 150 |
+
"policy_rollout_success_rate": 0.23008849557522124,
|
| 151 |
+
"restore_max_error": 4.76837158203125e-07
|
| 152 |
+
},
|
| 153 |
+
"PickCube-v1": {
|
| 154 |
+
"action_mse_to_best": 0.32018994461016165,
|
| 155 |
+
"expert_success_rate": 0.9402173913043478,
|
| 156 |
+
"num_groups": 184,
|
| 157 |
+
"oracle_success_rate": 0.9456521739130435,
|
| 158 |
+
"policy_expert_regret": 1.0790397456199254,
|
| 159 |
+
"policy_oracle_regret": 1.0790637500948557,
|
| 160 |
+
"policy_rollout_progress": 0.5715968862135449,
|
| 161 |
+
"policy_rollout_success_rate": 0.28804347826086957,
|
| 162 |
+
"restore_max_error": 4.76837158203125e-07
|
| 163 |
+
},
|
| 164 |
+
"PullCube-v1": {
|
| 165 |
+
"action_mse_to_best": 0.5818546704555813,
|
| 166 |
+
"expert_success_rate": 0.25,
|
| 167 |
+
"num_groups": 76,
|
| 168 |
+
"oracle_success_rate": 0.40789473684210525,
|
| 169 |
+
"policy_expert_regret": 0.301263104462506,
|
| 170 |
+
"policy_oracle_regret": 0.5490012017176732,
|
| 171 |
+
"policy_rollout_progress": 0.27730434748197075,
|
| 172 |
+
"policy_rollout_success_rate": 0.17105263157894737,
|
| 173 |
+
"restore_max_error": 4.76837158203125e-07
|
| 174 |
+
},
|
| 175 |
+
"PushCube-v1": {
|
| 176 |
+
"action_mse_to_best": 0.3549786427872138,
|
| 177 |
+
"expert_success_rate": 0.8198198198198198,
|
| 178 |
+
"num_groups": 111,
|
| 179 |
+
"oracle_success_rate": 1.0,
|
| 180 |
+
"policy_expert_regret": 0.3476062170020095,
|
| 181 |
+
"policy_oracle_regret": 0.381086724030005,
|
| 182 |
+
"policy_rollout_progress": 0.8171114741681932,
|
| 183 |
+
"policy_rollout_success_rate": 0.8018018018018018,
|
| 184 |
+
"restore_max_error": 4.76837158203125e-07
|
| 185 |
+
},
|
| 186 |
+
"StackCube-v1": {
|
| 187 |
+
"action_mse_to_best": 0.5013335423625924,
|
| 188 |
+
"expert_success_rate": 0.6923076923076923,
|
| 189 |
+
"num_groups": 91,
|
| 190 |
+
"oracle_success_rate": 0.8571428571428571,
|
| 191 |
+
"policy_expert_regret": 1.1277546356995027,
|
| 192 |
+
"policy_oracle_regret": 1.2999023898289754,
|
| 193 |
+
"policy_rollout_progress": 0.3907029373305185,
|
| 194 |
+
"policy_rollout_success_rate": 0.15384615384615385,
|
| 195 |
+
"restore_max_error": 3.948807716369629e-07
|
| 196 |
+
}
|
| 197 |
+
}
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"seed": 2,
|
| 201 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p20.json",
|
| 202 |
+
"num_groups": 575,
|
| 203 |
+
"selection_mode": "retrieval_residual",
|
| 204 |
+
"num_candidates": 32,
|
| 205 |
+
"candidate_sigma": 0.0,
|
| 206 |
+
"selection_margin": 0.2,
|
| 207 |
+
"field_optim_steps": 0,
|
| 208 |
+
"field_optim_step_size": 0.0,
|
| 209 |
+
"field_optim_trust_radius": 0.0,
|
| 210 |
+
"field_optim_l2_penalty": 0.0,
|
| 211 |
+
"retrieval_neighbors": 2,
|
| 212 |
+
"retrieval_metric": "raw",
|
| 213 |
+
"retrieval_type_min_success": 0.0,
|
| 214 |
+
"retrieval_residual_scale": 0.35,
|
| 215 |
+
"retrieval_residual_anchor": "expert",
|
| 216 |
+
"policy_rollout_success_rate": 0.37217391304347824,
|
| 217 |
+
"policy_rollout_progress": 0.5836175643415134,
|
| 218 |
+
"oracle_success_rate": 0.8765217391304347,
|
| 219 |
+
"action_mse_to_best": 0.41959726772635525,
|
| 220 |
+
"best_policy_val": {
|
| 221 |
+
"bc_loss": 0.11367896075050037,
|
| 222 |
+
"field_effect_loss": 0.009670218582161598,
|
| 223 |
+
"field_potential_loss": 0.2641640139950646,
|
| 224 |
+
"field_preference_loss": 0.5130490180518892,
|
| 225 |
+
"lattice_edges": 3833.3333333333335,
|
| 226 |
+
"progress_mae": 0.2021729110015763,
|
| 227 |
+
"rank_acc": 0.8333857821093665,
|
| 228 |
+
"rank_loss": 0.5130119257503085,
|
| 229 |
+
"regret_mae": 0.3958987047274907,
|
| 230 |
+
"success_accuracy": 0.8680730561415354,
|
| 231 |
+
"total_loss": 1.4394984311527677
|
| 232 |
+
},
|
| 233 |
+
"per_task": {
|
| 234 |
+
"LiftPegUpright-v1": {
|
| 235 |
+
"action_mse_to_best": 0.35413535257976037,
|
| 236 |
+
"expert_success_rate": 0.8229166666666666,
|
| 237 |
+
"num_groups": 96,
|
| 238 |
+
"oracle_success_rate": 0.9270833333333334,
|
| 239 |
+
"policy_expert_regret": 0.8470402403424183,
|
| 240 |
+
"policy_oracle_regret": 0.9533149556567272,
|
| 241 |
+
"policy_rollout_progress": 0.6469602342694998,
|
| 242 |
+
"policy_rollout_success_rate": 0.3333333333333333,
|
| 243 |
+
"restore_max_error": 3.5762786865234375e-07
|
| 244 |
+
},
|
| 245 |
+
"PickCube-v1": {
|
| 246 |
+
"action_mse_to_best": 0.3367153564373953,
|
| 247 |
+
"expert_success_rate": 0.9444444444444444,
|
| 248 |
+
"num_groups": 198,
|
| 249 |
+
"oracle_success_rate": 0.9595959595959596,
|
| 250 |
+
"policy_expert_regret": 0.9762676360668859,
|
| 251 |
+
"policy_oracle_regret": 0.9833510865955942,
|
| 252 |
+
"policy_rollout_progress": 0.633064233575656,
|
| 253 |
+
"policy_rollout_success_rate": 0.3383838383838384,
|
| 254 |
+
"restore_max_error": 4.76837158203125e-07
|
| 255 |
+
},
|
| 256 |
+
"PullCube-v1": {
|
| 257 |
+
"action_mse_to_best": 0.647159209019608,
|
| 258 |
+
"expert_success_rate": 0.24444444444444444,
|
| 259 |
+
"num_groups": 90,
|
| 260 |
+
"oracle_success_rate": 0.4666666666666667,
|
| 261 |
+
"policy_expert_regret": 0.27411871666984855,
|
| 262 |
+
"policy_oracle_regret": 0.5019587218830364,
|
| 263 |
+
"policy_rollout_progress": 0.33744355530513226,
|
| 264 |
+
"policy_rollout_success_rate": 0.23333333333333334,
|
| 265 |
+
"restore_max_error": 4.0978193283081055e-07
|
| 266 |
+
},
|
| 267 |
+
"PushCube-v1": {
|
| 268 |
+
"action_mse_to_best": 0.38244653175963034,
|
| 269 |
+
"expert_success_rate": 0.8514851485148515,
|
| 270 |
+
"num_groups": 101,
|
| 271 |
+
"oracle_success_rate": 1.0,
|
| 272 |
+
"policy_expert_regret": 0.36179918213055867,
|
| 273 |
+
"policy_oracle_regret": 0.41670241848667067,
|
| 274 |
+
"policy_rollout_progress": 0.8011193636915471,
|
| 275 |
+
"policy_rollout_success_rate": 0.7821782178217822,
|
| 276 |
+
"restore_max_error": 4.76837158203125e-07
|
| 277 |
+
},
|
| 278 |
+
"StackCube-v1": {
|
| 279 |
+
"action_mse_to_best": 0.4858929555656181,
|
| 280 |
+
"expert_success_rate": 0.7666666666666667,
|
| 281 |
+
"num_groups": 90,
|
| 282 |
+
"oracle_success_rate": 0.9111111111111111,
|
| 283 |
+
"policy_expert_regret": 1.1569180660777623,
|
| 284 |
+
"policy_oracle_regret": 1.3059902312027083,
|
| 285 |
+
"policy_rollout_progress": 0.4093580338690016,
|
| 286 |
+
"policy_rollout_success_rate": 0.16666666666666666,
|
| 287 |
+
"restore_max_error": 4.76837158203125e-07
|
| 288 |
+
}
|
| 289 |
+
}
|
| 290 |
+
}
|
| 291 |
+
]
|
| 292 |
+
}
|
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p35_safe_types_margin0p20_summary.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# h=16 Best-Policy Checkpoint Rollout
|
| 2 |
+
|
| 3 |
+
Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
|
| 4 |
+
Objective: `near_miss_policy_bc5`
|
| 5 |
+
Result file: `policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p20.json`
|
| 6 |
+
Completed seeds: 3
|
| 7 |
+
Baseline h=4 policy success: 29.67%
|
| 8 |
+
Baseline h=16 rank-checkpoint success: 29.74%
|
| 9 |
+
|
| 10 |
+
Mean success: 35.01% +/- 1.91%
|
| 11 |
+
Gain vs h=16 rank checkpoint: +5.28%
|
| 12 |
+
Mean progress: 56.66%
|
| 13 |
+
Mean action MSE to best: 0.397
|
| 14 |
+
|
| 15 |
+
| seed | mode | k | retrieval K | retrieval metric | residual anchor | min type success | residual scale | margin | sigma | opt steps | trust | success | progress | oracle | action MSE |
|
| 16 |
+
|---:|---|---:|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
| 17 |
+
| 0 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.35 | 0.200 | 0.00 | 0 | 0.00 | 33.91% | 55.23% | 85.74% | 0.381 |
|
| 18 |
+
| 1 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.35 | 0.200 | 0.00 | 0 | 0.00 | 33.91% | 56.40% | 86.96% | 0.391 |
|
| 19 |
+
| 2 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.35 | 0.200 | 0.00 | 0 | 0.00 | 37.22% | 58.36% | 87.65% | 0.420 |
|
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p35_safe_types_margin0p25_summary.json
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
|
| 3 |
+
"objective": "near_miss_policy_bc5",
|
| 4 |
+
"out_name": "policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p25.json",
|
| 5 |
+
"num_completed": 3,
|
| 6 |
+
"baseline_h4_policy_success": 0.2967,
|
| 7 |
+
"baseline_h16_rank_checkpoint_success": 0.29739130434782607,
|
| 8 |
+
"mean_success": 0.3466666666666667,
|
| 9 |
+
"std_success": 0.01759303293390554,
|
| 10 |
+
"mean_progress": 0.5645089878962977,
|
| 11 |
+
"mean_action_mse_to_best": 0.3954999150443768,
|
| 12 |
+
"gain_vs_h4": 0.04996666666666666,
|
| 13 |
+
"gain_vs_h16_rank_checkpoint": 0.04927536231884061,
|
| 14 |
+
"rows": [
|
| 15 |
+
{
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p25.json",
|
| 18 |
+
"num_groups": 575,
|
| 19 |
+
"selection_mode": "retrieval_residual",
|
| 20 |
+
"num_candidates": 32,
|
| 21 |
+
"candidate_sigma": 0.0,
|
| 22 |
+
"selection_margin": 0.25,
|
| 23 |
+
"field_optim_steps": 0,
|
| 24 |
+
"field_optim_step_size": 0.0,
|
| 25 |
+
"field_optim_trust_radius": 0.0,
|
| 26 |
+
"field_optim_l2_penalty": 0.0,
|
| 27 |
+
"retrieval_neighbors": 2,
|
| 28 |
+
"retrieval_metric": "raw",
|
| 29 |
+
"retrieval_type_min_success": 0.0,
|
| 30 |
+
"retrieval_residual_scale": 0.35,
|
| 31 |
+
"retrieval_residual_anchor": "expert",
|
| 32 |
+
"policy_rollout_success_rate": 0.3373913043478261,
|
| 33 |
+
"policy_rollout_progress": 0.5502470169001785,
|
| 34 |
+
"oracle_success_rate": 0.8573913043478261,
|
| 35 |
+
"action_mse_to_best": 0.38020374958848824,
|
| 36 |
+
"best_policy_val": {
|
| 37 |
+
"bc_loss": 0.13721593966086706,
|
| 38 |
+
"field_effect_loss": 0.009290305380192068,
|
| 39 |
+
"field_potential_loss": 0.2666468388504452,
|
| 40 |
+
"field_preference_loss": 0.5130573478009965,
|
| 41 |
+
"lattice_edges": 3833.3333333333335,
|
| 42 |
+
"progress_mae": 0.1933159919248687,
|
| 43 |
+
"rank_acc": 0.8265031774838766,
|
| 44 |
+
"rank_loss": 0.5130523675017886,
|
| 45 |
+
"regret_mae": 0.3756548762321472,
|
| 46 |
+
"success_accuracy": 0.8773836526605818,
|
| 47 |
+
"total_loss": 1.5581054819954767
|
| 48 |
+
},
|
| 49 |
+
"per_task": {
|
| 50 |
+
"LiftPegUpright-v1": {
|
| 51 |
+
"action_mse_to_best": 0.31969772514973566,
|
| 52 |
+
"expert_success_rate": 0.8865979381443299,
|
| 53 |
+
"num_groups": 97,
|
| 54 |
+
"oracle_success_rate": 0.9278350515463918,
|
| 55 |
+
"policy_expert_regret": 1.0792337135248578,
|
| 56 |
+
"policy_oracle_regret": 1.1056491912333006,
|
| 57 |
+
"policy_rollout_progress": 0.570563180084081,
|
| 58 |
+
"policy_rollout_success_rate": 0.2268041237113402,
|
| 59 |
+
"restore_max_error": 4.76837158203125e-07
|
| 60 |
+
},
|
| 61 |
+
"PickCube-v1": {
|
| 62 |
+
"action_mse_to_best": 0.30274310833416307,
|
| 63 |
+
"expert_success_rate": 0.9375,
|
| 64 |
+
"num_groups": 208,
|
| 65 |
+
"oracle_success_rate": 0.9471153846153846,
|
| 66 |
+
"policy_expert_regret": 0.9681778949542115,
|
| 67 |
+
"policy_oracle_regret": 0.9816007623807169,
|
| 68 |
+
"policy_rollout_progress": 0.613896441180259,
|
| 69 |
+
"policy_rollout_success_rate": 0.33653846153846156,
|
| 70 |
+
"restore_max_error": 4.76837158203125e-07
|
| 71 |
+
},
|
| 72 |
+
"PullCube-v1": {
|
| 73 |
+
"action_mse_to_best": 0.6023112506448448,
|
| 74 |
+
"expert_success_rate": 0.19480519480519481,
|
| 75 |
+
"num_groups": 77,
|
| 76 |
+
"oracle_success_rate": 0.36363636363636365,
|
| 77 |
+
"policy_expert_regret": 0.19716023989082254,
|
| 78 |
+
"policy_oracle_regret": 0.39124274369051026,
|
| 79 |
+
"policy_rollout_progress": 0.3352508790784987,
|
| 80 |
+
"policy_rollout_success_rate": 0.23376623376623376,
|
| 81 |
+
"restore_max_error": 4.76837158203125e-07
|
| 82 |
+
},
|
| 83 |
+
"PushCube-v1": {
|
| 84 |
+
"action_mse_to_best": 0.3656082003912393,
|
| 85 |
+
"expert_success_rate": 0.8617021276595744,
|
| 86 |
+
"num_groups": 94,
|
| 87 |
+
"oracle_success_rate": 0.9893617021276596,
|
| 88 |
+
"policy_expert_regret": 0.5719942746961371,
|
| 89 |
+
"policy_oracle_regret": 0.66845020953011,
|
| 90 |
+
"policy_rollout_progress": 0.682613620257124,
|
| 91 |
+
"policy_rollout_success_rate": 0.648936170212766,
|
| 92 |
+
"restore_max_error": 4.76837158203125e-07
|
| 93 |
+
},
|
| 94 |
+
"StackCube-v1": {
|
| 95 |
+
"action_mse_to_best": 0.4433411414537466,
|
| 96 |
+
"expert_success_rate": 0.6767676767676768,
|
| 97 |
+
"num_groups": 99,
|
| 98 |
+
"oracle_success_rate": 0.8585858585858586,
|
| 99 |
+
"policy_expert_regret": 0.9762592216332754,
|
| 100 |
+
"policy_oracle_regret": 1.1504355197904086,
|
| 101 |
+
"policy_rollout_progress": 0.43815099485594816,
|
| 102 |
+
"policy_rollout_success_rate": 0.23232323232323232,
|
| 103 |
+
"restore_max_error": 3.948807716369629e-07
|
| 104 |
+
}
|
| 105 |
+
}
|
| 106 |
+
},
|
| 107 |
+
{
|
| 108 |
+
"seed": 1,
|
| 109 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p25.json",
|
| 110 |
+
"num_groups": 575,
|
| 111 |
+
"selection_mode": "retrieval_residual",
|
| 112 |
+
"num_candidates": 32,
|
| 113 |
+
"candidate_sigma": 0.0,
|
| 114 |
+
"selection_margin": 0.25,
|
| 115 |
+
"field_optim_steps": 0,
|
| 116 |
+
"field_optim_step_size": 0.0,
|
| 117 |
+
"field_optim_trust_radius": 0.0,
|
| 118 |
+
"field_optim_l2_penalty": 0.0,
|
| 119 |
+
"retrieval_neighbors": 2,
|
| 120 |
+
"retrieval_metric": "raw",
|
| 121 |
+
"retrieval_type_min_success": 0.0,
|
| 122 |
+
"retrieval_residual_scale": 0.35,
|
| 123 |
+
"retrieval_residual_anchor": "expert",
|
| 124 |
+
"policy_rollout_success_rate": 0.3356521739130435,
|
| 125 |
+
"policy_rollout_progress": 0.5607895659104637,
|
| 126 |
+
"oracle_success_rate": 0.8695652173913043,
|
| 127 |
+
"action_mse_to_best": 0.3891039643112732,
|
| 128 |
+
"best_policy_val": {
|
| 129 |
+
"bc_loss": 0.1315989200439718,
|
| 130 |
+
"field_effect_loss": 0.010110370814800262,
|
| 131 |
+
"field_potential_loss": 0.31546536915832096,
|
| 132 |
+
"field_preference_loss": 0.4960637440284093,
|
| 133 |
+
"lattice_edges": 3833.3333333333335,
|
| 134 |
+
"progress_mae": 0.19023226698239645,
|
| 135 |
+
"rank_acc": 0.8255469501018524,
|
| 136 |
+
"rank_loss": 0.4960531195004781,
|
| 137 |
+
"regret_mae": 0.42156532737943864,
|
| 138 |
+
"success_accuracy": 0.8807473679383596,
|
| 139 |
+
"total_loss": 1.561984618504842
|
| 140 |
+
},
|
| 141 |
+
"per_task": {
|
| 142 |
+
"LiftPegUpright-v1": {
|
| 143 |
+
"action_mse_to_best": 0.32557108186482064,
|
| 144 |
+
"expert_success_rate": 0.8584070796460177,
|
| 145 |
+
"num_groups": 113,
|
| 146 |
+
"oracle_success_rate": 0.9380530973451328,
|
| 147 |
+
"policy_expert_regret": 0.987694250535121,
|
| 148 |
+
"policy_oracle_regret": 1.066896405631462,
|
| 149 |
+
"policy_rollout_progress": 0.6352153201546289,
|
| 150 |
+
"policy_rollout_success_rate": 0.23008849557522124,
|
| 151 |
+
"restore_max_error": 4.76837158203125e-07
|
| 152 |
+
},
|
| 153 |
+
"PickCube-v1": {
|
| 154 |
+
"action_mse_to_best": 0.31980027794918936,
|
| 155 |
+
"expert_success_rate": 0.9402173913043478,
|
| 156 |
+
"num_groups": 184,
|
| 157 |
+
"oracle_success_rate": 0.9456521739130435,
|
| 158 |
+
"policy_expert_regret": 1.079014161105632,
|
| 159 |
+
"policy_oracle_regret": 1.0790381655805623,
|
| 160 |
+
"policy_rollout_progress": 0.5716224707278382,
|
| 161 |
+
"policy_rollout_success_rate": 0.28804347826086957,
|
| 162 |
+
"restore_max_error": 4.76837158203125e-07
|
| 163 |
+
},
|
| 164 |
+
"PullCube-v1": {
|
| 165 |
+
"action_mse_to_best": 0.569444256393533,
|
| 166 |
+
"expert_success_rate": 0.25,
|
| 167 |
+
"num_groups": 76,
|
| 168 |
+
"oracle_success_rate": 0.40789473684210525,
|
| 169 |
+
"policy_expert_regret": 0.32589247040940744,
|
| 170 |
+
"policy_oracle_regret": 0.5741651183925569,
|
| 171 |
+
"policy_rollout_progress": 0.26474735648126196,
|
| 172 |
+
"policy_rollout_success_rate": 0.15789473684210525,
|
| 173 |
+
"restore_max_error": 4.76837158203125e-07
|
| 174 |
+
},
|
| 175 |
+
"PushCube-v1": {
|
| 176 |
+
"action_mse_to_best": 0.3542353541397297,
|
| 177 |
+
"expert_success_rate": 0.8198198198198198,
|
| 178 |
+
"num_groups": 111,
|
| 179 |
+
"oracle_success_rate": 1.0,
|
| 180 |
+
"policy_expert_regret": 0.34830493110794203,
|
| 181 |
+
"policy_oracle_regret": 0.39806604143735524,
|
| 182 |
+
"policy_rollout_progress": 0.809141165769852,
|
| 183 |
+
"policy_rollout_success_rate": 0.7927927927927928,
|
| 184 |
+
"restore_max_error": 4.76837158203125e-07
|
| 185 |
+
},
|
| 186 |
+
"StackCube-v1": {
|
| 187 |
+
"action_mse_to_best": 0.5000451460460221,
|
| 188 |
+
"expert_success_rate": 0.6923076923076923,
|
| 189 |
+
"num_groups": 91,
|
| 190 |
+
"oracle_success_rate": 0.8571428571428571,
|
| 191 |
+
"policy_expert_regret": 1.1276810223257148,
|
| 192 |
+
"policy_oracle_regret": 1.2998287764551875,
|
| 193 |
+
"policy_rollout_progress": 0.3907765507043063,
|
| 194 |
+
"policy_rollout_success_rate": 0.15384615384615385,
|
| 195 |
+
"restore_max_error": 3.948807716369629e-07
|
| 196 |
+
}
|
| 197 |
+
}
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"seed": 2,
|
| 201 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p25.json",
|
| 202 |
+
"num_groups": 575,
|
| 203 |
+
"selection_mode": "retrieval_residual",
|
| 204 |
+
"num_candidates": 32,
|
| 205 |
+
"candidate_sigma": 0.0,
|
| 206 |
+
"selection_margin": 0.25,
|
| 207 |
+
"field_optim_steps": 0,
|
| 208 |
+
"field_optim_step_size": 0.0,
|
| 209 |
+
"field_optim_trust_radius": 0.0,
|
| 210 |
+
"field_optim_l2_penalty": 0.0,
|
| 211 |
+
"retrieval_neighbors": 2,
|
| 212 |
+
"retrieval_metric": "raw",
|
| 213 |
+
"retrieval_type_min_success": 0.0,
|
| 214 |
+
"retrieval_residual_scale": 0.35,
|
| 215 |
+
"retrieval_residual_anchor": "expert",
|
| 216 |
+
"policy_rollout_success_rate": 0.36695652173913046,
|
| 217 |
+
"policy_rollout_progress": 0.5824903808782509,
|
| 218 |
+
"oracle_success_rate": 0.8765217391304347,
|
| 219 |
+
"action_mse_to_best": 0.417192031233369,
|
| 220 |
+
"best_policy_val": {
|
| 221 |
+
"bc_loss": 0.11367896075050037,
|
| 222 |
+
"field_effect_loss": 0.009670218582161598,
|
| 223 |
+
"field_potential_loss": 0.2641640139950646,
|
| 224 |
+
"field_preference_loss": 0.5130490180518892,
|
| 225 |
+
"lattice_edges": 3833.3333333333335,
|
| 226 |
+
"progress_mae": 0.2021729110015763,
|
| 227 |
+
"rank_acc": 0.8333857821093665,
|
| 228 |
+
"rank_loss": 0.5130119257503085,
|
| 229 |
+
"regret_mae": 0.3958987047274907,
|
| 230 |
+
"success_accuracy": 0.8680730561415354,
|
| 231 |
+
"total_loss": 1.4394984311527677
|
| 232 |
+
},
|
| 233 |
+
"per_task": {
|
| 234 |
+
"LiftPegUpright-v1": {
|
| 235 |
+
"action_mse_to_best": 0.35361564260771655,
|
| 236 |
+
"expert_success_rate": 0.8229166666666666,
|
| 237 |
+
"num_groups": 96,
|
| 238 |
+
"oracle_success_rate": 0.9270833333333334,
|
| 239 |
+
"policy_expert_regret": 0.8470402403424183,
|
| 240 |
+
"policy_oracle_regret": 0.9533149556567272,
|
| 241 |
+
"policy_rollout_progress": 0.6469602342694998,
|
| 242 |
+
"policy_rollout_success_rate": 0.3333333333333333,
|
| 243 |
+
"restore_max_error": 3.5762786865234375e-07
|
| 244 |
+
},
|
| 245 |
+
"PickCube-v1": {
|
| 246 |
+
"action_mse_to_best": 0.33547671381948574,
|
| 247 |
+
"expert_success_rate": 0.9444444444444444,
|
| 248 |
+
"num_groups": 198,
|
| 249 |
+
"oracle_success_rate": 0.9595959595959596,
|
| 250 |
+
"policy_expert_regret": 0.9958049548281864,
|
| 251 |
+
"policy_oracle_regret": 1.002888405356895,
|
| 252 |
+
"policy_rollout_progress": 0.6286784299658705,
|
| 253 |
+
"policy_rollout_success_rate": 0.32323232323232326,
|
| 254 |
+
"restore_max_error": 4.76837158203125e-07
|
| 255 |
+
},
|
| 256 |
+
"PullCube-v1": {
|
| 257 |
+
"action_mse_to_best": 0.6370052759846051,
|
| 258 |
+
"expert_success_rate": 0.24444444444444444,
|
| 259 |
+
"num_groups": 90,
|
| 260 |
+
"oracle_success_rate": 0.4666666666666667,
|
| 261 |
+
"policy_expert_regret": 0.2727692198284179,
|
| 262 |
+
"policy_oracle_regret": 0.4998151001071973,
|
| 263 |
+
"policy_rollout_progress": 0.33958717708097136,
|
| 264 |
+
"policy_rollout_success_rate": 0.23333333333333334,
|
| 265 |
+
"restore_max_error": 4.0978193283081055e-07
|
| 266 |
+
},
|
| 267 |
+
"PushCube-v1": {
|
| 268 |
+
"action_mse_to_best": 0.38118959904307187,
|
| 269 |
+
"expert_success_rate": 0.8514851485148515,
|
| 270 |
+
"num_groups": 101,
|
| 271 |
+
"oracle_success_rate": 1.0,
|
| 272 |
+
"policy_expert_regret": 0.36179918213055867,
|
| 273 |
+
"policy_oracle_regret": 0.41670241848667067,
|
| 274 |
+
"policy_rollout_progress": 0.8011193636915471,
|
| 275 |
+
"policy_rollout_success_rate": 0.7821782178217822,
|
| 276 |
+
"restore_max_error": 4.76837158203125e-07
|
| 277 |
+
},
|
| 278 |
+
"StackCube-v1": {
|
| 279 |
+
"action_mse_to_best": 0.48537002878470553,
|
| 280 |
+
"expert_success_rate": 0.7666666666666667,
|
| 281 |
+
"num_groups": 90,
|
| 282 |
+
"oracle_success_rate": 0.9111111111111111,
|
| 283 |
+
"policy_expert_regret": 1.1566143698162503,
|
| 284 |
+
"policy_oracle_regret": 1.3056865349411964,
|
| 285 |
+
"policy_rollout_progress": 0.4096617301305135,
|
| 286 |
+
"policy_rollout_success_rate": 0.16666666666666666,
|
| 287 |
+
"restore_max_error": 4.76837158203125e-07
|
| 288 |
+
}
|
| 289 |
+
}
|
| 290 |
+
}
|
| 291 |
+
]
|
| 292 |
+
}
|
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p35_safe_types_margin0p25_summary.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# h=16 Best-Policy Checkpoint Rollout
|
| 2 |
+
|
| 3 |
+
Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
|
| 4 |
+
Objective: `near_miss_policy_bc5`
|
| 5 |
+
Result file: `policy_rollout_retrieval_residual_knn2_scale0p35_safe_types_margin0p25.json`
|
| 6 |
+
Completed seeds: 3
|
| 7 |
+
Baseline h=4 policy success: 29.67%
|
| 8 |
+
Baseline h=16 rank-checkpoint success: 29.74%
|
| 9 |
+
|
| 10 |
+
Mean success: 34.67% +/- 1.76%
|
| 11 |
+
Gain vs h=16 rank checkpoint: +4.93%
|
| 12 |
+
Mean progress: 56.45%
|
| 13 |
+
Mean action MSE to best: 0.395
|
| 14 |
+
|
| 15 |
+
| seed | mode | k | retrieval K | retrieval metric | residual anchor | min type success | residual scale | margin | sigma | opt steps | trust | success | progress | oracle | action MSE |
|
| 16 |
+
|---:|---|---:|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
| 17 |
+
| 0 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.35 | 0.250 | 0.00 | 0 | 0.00 | 33.74% | 55.02% | 85.74% | 0.380 |
|
| 18 |
+
| 1 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.35 | 0.250 | 0.00 | 0 | 0.00 | 33.57% | 56.08% | 86.96% | 0.389 |
|
| 19 |
+
| 2 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.35 | 0.250 | 0.00 | 0 | 0.00 | 36.70% | 58.25% | 87.65% | 0.417 |
|
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p40_safe_types_margin0p20_summary.json
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
|
| 3 |
+
"objective": "near_miss_policy_bc5",
|
| 4 |
+
"out_name": "policy_rollout_retrieval_residual_knn2_scale0p40_safe_types_margin0p20.json",
|
| 5 |
+
"num_completed": 3,
|
| 6 |
+
"baseline_h4_policy_success": 0.2967,
|
| 7 |
+
"baseline_h16_rank_checkpoint_success": 0.29739130434782607,
|
| 8 |
+
"mean_success": 0.3501449275362319,
|
| 9 |
+
"std_success": 0.016159257814221867,
|
| 10 |
+
"mean_progress": 0.5670320824102236,
|
| 11 |
+
"mean_action_mse_to_best": 0.3984417730505052,
|
| 12 |
+
"gain_vs_h4": 0.05344492753623187,
|
| 13 |
+
"gain_vs_h16_rank_checkpoint": 0.05275362318840582,
|
| 14 |
+
"rows": [
|
| 15 |
+
{
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_knn2_scale0p40_safe_types_margin0p20.json",
|
| 18 |
+
"num_groups": 575,
|
| 19 |
+
"selection_mode": "retrieval_residual",
|
| 20 |
+
"num_candidates": 32,
|
| 21 |
+
"candidate_sigma": 0.0,
|
| 22 |
+
"selection_margin": 0.2,
|
| 23 |
+
"field_optim_steps": 0,
|
| 24 |
+
"field_optim_step_size": 0.0,
|
| 25 |
+
"field_optim_trust_radius": 0.0,
|
| 26 |
+
"field_optim_l2_penalty": 0.0,
|
| 27 |
+
"retrieval_neighbors": 2,
|
| 28 |
+
"retrieval_metric": "raw",
|
| 29 |
+
"retrieval_type_min_success": 0.0,
|
| 30 |
+
"retrieval_residual_scale": 0.4,
|
| 31 |
+
"retrieval_residual_anchor": "expert",
|
| 32 |
+
"policy_rollout_success_rate": 0.3426086956521739,
|
| 33 |
+
"policy_rollout_progress": 0.5547752405812397,
|
| 34 |
+
"oracle_success_rate": 0.8573913043478261,
|
| 35 |
+
"action_mse_to_best": 0.38253434254990326,
|
| 36 |
+
"best_policy_val": {
|
| 37 |
+
"bc_loss": 0.13721593966086706,
|
| 38 |
+
"field_effect_loss": 0.009290305380192068,
|
| 39 |
+
"field_potential_loss": 0.2666468388504452,
|
| 40 |
+
"field_preference_loss": 0.5130573478009965,
|
| 41 |
+
"lattice_edges": 3833.3333333333335,
|
| 42 |
+
"progress_mae": 0.1933159919248687,
|
| 43 |
+
"rank_acc": 0.8265031774838766,
|
| 44 |
+
"rank_loss": 0.5130523675017886,
|
| 45 |
+
"regret_mae": 0.3756548762321472,
|
| 46 |
+
"success_accuracy": 0.8773836526605818,
|
| 47 |
+
"total_loss": 1.5581054819954767
|
| 48 |
+
},
|
| 49 |
+
"per_task": {
|
| 50 |
+
"LiftPegUpright-v1": {
|
| 51 |
+
"action_mse_to_best": 0.31969772514973566,
|
| 52 |
+
"expert_success_rate": 0.8865979381443299,
|
| 53 |
+
"num_groups": 97,
|
| 54 |
+
"oracle_success_rate": 0.9278350515463918,
|
| 55 |
+
"policy_expert_regret": 1.0792337135248578,
|
| 56 |
+
"policy_oracle_regret": 1.1056491912333006,
|
| 57 |
+
"policy_rollout_progress": 0.570563180084081,
|
| 58 |
+
"policy_rollout_success_rate": 0.2268041237113402,
|
| 59 |
+
"restore_max_error": 4.76837158203125e-07
|
| 60 |
+
},
|
| 61 |
+
"PickCube-v1": {
|
| 62 |
+
"action_mse_to_best": 0.302895870203009,
|
| 63 |
+
"expert_success_rate": 0.9375,
|
| 64 |
+
"num_groups": 208,
|
| 65 |
+
"oracle_success_rate": 0.9471153846153846,
|
| 66 |
+
"policy_expert_regret": 0.9613472516648471,
|
| 67 |
+
"policy_oracle_regret": 0.9747701190913526,
|
| 68 |
+
"policy_rollout_progress": 0.615919392161931,
|
| 69 |
+
"policy_rollout_success_rate": 0.34134615384615385,
|
| 70 |
+
"restore_max_error": 4.76837158203125e-07
|
| 71 |
+
},
|
| 72 |
+
"PullCube-v1": {
|
| 73 |
+
"action_mse_to_best": 0.6015770019261868,
|
| 74 |
+
"expert_success_rate": 0.19480519480519481,
|
| 75 |
+
"num_groups": 77,
|
| 76 |
+
"oracle_success_rate": 0.36363636363636365,
|
| 77 |
+
"policy_expert_regret": 0.1972956603616718,
|
| 78 |
+
"policy_oracle_regret": 0.3680945536780909,
|
| 79 |
+
"policy_rollout_progress": 0.34545655677687714,
|
| 80 |
+
"policy_rollout_success_rate": 0.24675324675324675,
|
| 81 |
+
"restore_max_error": 4.76837158203125e-07
|
| 82 |
+
},
|
| 83 |
+
"PushCube-v1": {
|
| 84 |
+
"action_mse_to_best": 0.36507201238357007,
|
| 85 |
+
"expert_success_rate": 0.8617021276595744,
|
| 86 |
+
"num_groups": 94,
|
| 87 |
+
"oracle_success_rate": 0.9893617021276596,
|
| 88 |
+
"policy_expert_regret": 0.5719942746961371,
|
| 89 |
+
"policy_oracle_regret": 0.6495450359392674,
|
| 90 |
+
"policy_rollout_progress": 0.6908804959756263,
|
| 91 |
+
"policy_rollout_success_rate": 0.6595744680851063,
|
| 92 |
+
"restore_max_error": 4.76837158203125e-07
|
| 93 |
+
},
|
| 94 |
+
"StackCube-v1": {
|
| 95 |
+
"action_mse_to_best": 0.4576366496168905,
|
| 96 |
+
"expert_success_rate": 0.6767676767676768,
|
| 97 |
+
"num_groups": 99,
|
| 98 |
+
"oracle_success_rate": 0.8585858585858586,
|
| 99 |
+
"policy_expert_regret": 0.9699796022790851,
|
| 100 |
+
"policy_oracle_regret": 1.1441725772438627,
|
| 101 |
+
"policy_rollout_progress": 0.444413937402494,
|
| 102 |
+
"policy_rollout_success_rate": 0.23232323232323232,
|
| 103 |
+
"restore_max_error": 3.948807716369629e-07
|
| 104 |
+
}
|
| 105 |
+
}
|
| 106 |
+
},
|
| 107 |
+
{
|
| 108 |
+
"seed": 1,
|
| 109 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_knn2_scale0p40_safe_types_margin0p20.json",
|
| 110 |
+
"num_groups": 575,
|
| 111 |
+
"selection_mode": "retrieval_residual",
|
| 112 |
+
"num_candidates": 32,
|
| 113 |
+
"candidate_sigma": 0.0,
|
| 114 |
+
"selection_margin": 0.2,
|
| 115 |
+
"field_optim_steps": 0,
|
| 116 |
+
"field_optim_step_size": 0.0,
|
| 117 |
+
"field_optim_trust_radius": 0.0,
|
| 118 |
+
"field_optim_l2_penalty": 0.0,
|
| 119 |
+
"retrieval_neighbors": 2,
|
| 120 |
+
"retrieval_metric": "raw",
|
| 121 |
+
"retrieval_type_min_success": 0.0,
|
| 122 |
+
"retrieval_residual_scale": 0.4,
|
| 123 |
+
"retrieval_residual_anchor": "expert",
|
| 124 |
+
"policy_rollout_success_rate": 0.3391304347826087,
|
| 125 |
+
"policy_rollout_progress": 0.5637077170092126,
|
| 126 |
+
"oracle_success_rate": 0.8695652173913043,
|
| 127 |
+
"action_mse_to_best": 0.3923974754207808,
|
| 128 |
+
"best_policy_val": {
|
| 129 |
+
"bc_loss": 0.1315989200439718,
|
| 130 |
+
"field_effect_loss": 0.010110370814800262,
|
| 131 |
+
"field_potential_loss": 0.31546536915832096,
|
| 132 |
+
"field_preference_loss": 0.4960637440284093,
|
| 133 |
+
"lattice_edges": 3833.3333333333335,
|
| 134 |
+
"progress_mae": 0.19023226698239645,
|
| 135 |
+
"rank_acc": 0.8255469501018524,
|
| 136 |
+
"rank_loss": 0.4960531195004781,
|
| 137 |
+
"regret_mae": 0.42156532737943864,
|
| 138 |
+
"success_accuracy": 0.8807473679383596,
|
| 139 |
+
"total_loss": 1.561984618504842
|
| 140 |
+
},
|
| 141 |
+
"per_task": {
|
| 142 |
+
"LiftPegUpright-v1": {
|
| 143 |
+
"action_mse_to_best": 0.3261066819829619,
|
| 144 |
+
"expert_success_rate": 0.8584070796460177,
|
| 145 |
+
"num_groups": 113,
|
| 146 |
+
"oracle_success_rate": 0.9380530973451328,
|
| 147 |
+
"policy_expert_regret": 0.987694250535121,
|
| 148 |
+
"policy_oracle_regret": 1.066896405631462,
|
| 149 |
+
"policy_rollout_progress": 0.6352153201546289,
|
| 150 |
+
"policy_rollout_success_rate": 0.23008849557522124,
|
| 151 |
+
"restore_max_error": 4.76837158203125e-07
|
| 152 |
+
},
|
| 153 |
+
"PickCube-v1": {
|
| 154 |
+
"action_mse_to_best": 0.3204136875701015,
|
| 155 |
+
"expert_success_rate": 0.9402173913043478,
|
| 156 |
+
"num_groups": 184,
|
| 157 |
+
"oracle_success_rate": 0.9456521739130435,
|
| 158 |
+
"policy_expert_regret": 1.0791211031863224,
|
| 159 |
+
"policy_oracle_regret": 1.0791451076612524,
|
| 160 |
+
"policy_rollout_progress": 0.571515528647148,
|
| 161 |
+
"policy_rollout_success_rate": 0.28804347826086957,
|
| 162 |
+
"restore_max_error": 4.76837158203125e-07
|
| 163 |
+
},
|
| 164 |
+
"PullCube-v1": {
|
| 165 |
+
"action_mse_to_best": 0.5815515975026708,
|
| 166 |
+
"expert_success_rate": 0.25,
|
| 167 |
+
"num_groups": 76,
|
| 168 |
+
"oracle_success_rate": 0.40789473684210525,
|
| 169 |
+
"policy_expert_regret": 0.3028959449576704,
|
| 170 |
+
"policy_oracle_regret": 0.5505132665367503,
|
| 171 |
+
"policy_rollout_progress": 0.27524131360022647,
|
| 172 |
+
"policy_rollout_success_rate": 0.17105263157894737,
|
| 173 |
+
"restore_max_error": 4.76837158203125e-07
|
| 174 |
+
},
|
| 175 |
+
"PushCube-v1": {
|
| 176 |
+
"action_mse_to_best": 0.3578707762733773,
|
| 177 |
+
"expert_success_rate": 0.8198198198198198,
|
| 178 |
+
"num_groups": 111,
|
| 179 |
+
"oracle_success_rate": 1.0,
|
| 180 |
+
"policy_expert_regret": 0.3474968559033162,
|
| 181 |
+
"policy_oracle_regret": 0.3809773629313117,
|
| 182 |
+
"policy_rollout_progress": 0.8172208352668865,
|
| 183 |
+
"policy_rollout_success_rate": 0.8018018018018018,
|
| 184 |
+
"restore_max_error": 4.76837158203125e-07
|
| 185 |
+
},
|
| 186 |
+
"StackCube-v1": {
|
| 187 |
+
"action_mse_to_best": 0.5044043649827222,
|
| 188 |
+
"expert_success_rate": 0.6923076923076923,
|
| 189 |
+
"num_groups": 91,
|
| 190 |
+
"oracle_success_rate": 0.8571428571428571,
|
| 191 |
+
"policy_expert_regret": 1.1276455259585119,
|
| 192 |
+
"policy_oracle_regret": 1.2997932800879846,
|
| 193 |
+
"policy_rollout_progress": 0.3908120470715093,
|
| 194 |
+
"policy_rollout_success_rate": 0.15384615384615385,
|
| 195 |
+
"restore_max_error": 3.948807716369629e-07
|
| 196 |
+
}
|
| 197 |
+
}
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"seed": 2,
|
| 201 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_knn2_scale0p40_safe_types_margin0p20.json",
|
| 202 |
+
"num_groups": 575,
|
| 203 |
+
"selection_mode": "retrieval_residual",
|
| 204 |
+
"num_candidates": 32,
|
| 205 |
+
"candidate_sigma": 0.0,
|
| 206 |
+
"selection_margin": 0.2,
|
| 207 |
+
"field_optim_steps": 0,
|
| 208 |
+
"field_optim_step_size": 0.0,
|
| 209 |
+
"field_optim_trust_radius": 0.0,
|
| 210 |
+
"field_optim_l2_penalty": 0.0,
|
| 211 |
+
"retrieval_neighbors": 2,
|
| 212 |
+
"retrieval_metric": "raw",
|
| 213 |
+
"retrieval_type_min_success": 0.0,
|
| 214 |
+
"retrieval_residual_scale": 0.4,
|
| 215 |
+
"retrieval_residual_anchor": "expert",
|
| 216 |
+
"policy_rollout_success_rate": 0.36869565217391304,
|
| 217 |
+
"policy_rollout_progress": 0.5826132896402186,
|
| 218 |
+
"oracle_success_rate": 0.8765217391304347,
|
| 219 |
+
"action_mse_to_best": 0.4203935011808315,
|
| 220 |
+
"best_policy_val": {
|
| 221 |
+
"bc_loss": 0.11367896075050037,
|
| 222 |
+
"field_effect_loss": 0.009670218582161598,
|
| 223 |
+
"field_potential_loss": 0.2641640139950646,
|
| 224 |
+
"field_preference_loss": 0.5130490180518892,
|
| 225 |
+
"lattice_edges": 3833.3333333333335,
|
| 226 |
+
"progress_mae": 0.2021729110015763,
|
| 227 |
+
"rank_acc": 0.8333857821093665,
|
| 228 |
+
"rank_loss": 0.5130119257503085,
|
| 229 |
+
"regret_mae": 0.3958987047274907,
|
| 230 |
+
"success_accuracy": 0.8680730561415354,
|
| 231 |
+
"total_loss": 1.4394984311527677
|
| 232 |
+
},
|
| 233 |
+
"per_task": {
|
| 234 |
+
"LiftPegUpright-v1": {
|
| 235 |
+
"action_mse_to_best": 0.354290255427865,
|
| 236 |
+
"expert_success_rate": 0.8229166666666666,
|
| 237 |
+
"num_groups": 96,
|
| 238 |
+
"oracle_success_rate": 0.9270833333333334,
|
| 239 |
+
"policy_expert_regret": 0.8470402403424183,
|
| 240 |
+
"policy_oracle_regret": 0.9533149556567272,
|
| 241 |
+
"policy_rollout_progress": 0.6469602342694998,
|
| 242 |
+
"policy_rollout_success_rate": 0.3333333333333333,
|
| 243 |
+
"restore_max_error": 3.5762786865234375e-07
|
| 244 |
+
},
|
| 245 |
+
"PickCube-v1": {
|
| 246 |
+
"action_mse_to_best": 0.3372542926044476,
|
| 247 |
+
"expert_success_rate": 0.9444444444444444,
|
| 248 |
+
"num_groups": 198,
|
| 249 |
+
"oracle_success_rate": 0.9595959595959596,
|
| 250 |
+
"policy_expert_regret": 0.9764947270259562,
|
| 251 |
+
"policy_oracle_regret": 0.9835781775546646,
|
| 252 |
+
"policy_rollout_progress": 0.6328371426165856,
|
| 253 |
+
"policy_rollout_success_rate": 0.3383838383838384,
|
| 254 |
+
"restore_max_error": 4.76837158203125e-07
|
| 255 |
+
},
|
| 256 |
+
"PullCube-v1": {
|
| 257 |
+
"action_mse_to_best": 0.642058200803068,
|
| 258 |
+
"expert_success_rate": 0.24444444444444444,
|
| 259 |
+
"num_groups": 90,
|
| 260 |
+
"oracle_success_rate": 0.4666666666666667,
|
| 261 |
+
"policy_expert_regret": 0.31410154858258466,
|
| 262 |
+
"policy_oracle_regret": 0.5195995818685535,
|
| 263 |
+
"policy_rollout_progress": 0.33091380643072626,
|
| 264 |
+
"policy_rollout_success_rate": 0.2222222222222222,
|
| 265 |
+
"restore_max_error": 4.0978193283081055e-07
|
| 266 |
+
},
|
| 267 |
+
"PushCube-v1": {
|
| 268 |
+
"action_mse_to_best": 0.3872983108947773,
|
| 269 |
+
"expert_success_rate": 0.8514851485148515,
|
| 270 |
+
"num_groups": 101,
|
| 271 |
+
"oracle_success_rate": 1.0,
|
| 272 |
+
"policy_expert_regret": 0.3612374585158754,
|
| 273 |
+
"policy_oracle_regret": 0.41614069487198746,
|
| 274 |
+
"policy_rollout_progress": 0.8016810873062303,
|
| 275 |
+
"policy_rollout_success_rate": 0.7821782178217822,
|
| 276 |
+
"restore_max_error": 4.76837158203125e-07
|
| 277 |
+
},
|
| 278 |
+
"StackCube-v1": {
|
| 279 |
+
"action_mse_to_best": 0.48928534721748695,
|
| 280 |
+
"expert_success_rate": 0.7666666666666667,
|
| 281 |
+
"num_groups": 90,
|
| 282 |
+
"oracle_success_rate": 0.9111111111111111,
|
| 283 |
+
"policy_expert_regret": 1.1551312858859697,
|
| 284 |
+
"policy_oracle_regret": 1.317118571533097,
|
| 285 |
+
"policy_rollout_progress": 0.4093408046497239,
|
| 286 |
+
"policy_rollout_success_rate": 0.15555555555555556,
|
| 287 |
+
"restore_max_error": 4.76837158203125e-07
|
| 288 |
+
}
|
| 289 |
+
}
|
| 290 |
+
}
|
| 291 |
+
]
|
| 292 |
+
}
|
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p40_safe_types_margin0p20_summary.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# h=16 Best-Policy Checkpoint Rollout
|
| 2 |
+
|
| 3 |
+
Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
|
| 4 |
+
Objective: `near_miss_policy_bc5`
|
| 5 |
+
Result file: `policy_rollout_retrieval_residual_knn2_scale0p40_safe_types_margin0p20.json`
|
| 6 |
+
Completed seeds: 3
|
| 7 |
+
Baseline h=4 policy success: 29.67%
|
| 8 |
+
Baseline h=16 rank-checkpoint success: 29.74%
|
| 9 |
+
|
| 10 |
+
Mean success: 35.01% +/- 1.62%
|
| 11 |
+
Gain vs h=16 rank checkpoint: +5.28%
|
| 12 |
+
Mean progress: 56.70%
|
| 13 |
+
Mean action MSE to best: 0.398
|
| 14 |
+
|
| 15 |
+
| seed | mode | k | retrieval K | retrieval metric | residual anchor | min type success | residual scale | margin | sigma | opt steps | trust | success | progress | oracle | action MSE |
|
| 16 |
+
|---:|---|---:|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
| 17 |
+
| 0 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.40 | 0.200 | 0.00 | 0 | 0.00 | 34.26% | 55.48% | 85.74% | 0.383 |
|
| 18 |
+
| 1 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.40 | 0.200 | 0.00 | 0 | 0.00 | 33.91% | 56.37% | 86.96% | 0.392 |
|
| 19 |
+
| 2 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.40 | 0.200 | 0.00 | 0 | 0.00 | 36.87% | 58.26% | 87.65% | 0.420 |
|
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p50_safe_types_margin0p20_summary.json
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
|
| 3 |
+
"objective": "near_miss_policy_bc5",
|
| 4 |
+
"out_name": "policy_rollout_retrieval_residual_knn2_scale0p50_safe_types_margin0p20.json",
|
| 5 |
+
"num_completed": 3,
|
| 6 |
+
"baseline_h4_policy_success": 0.2967,
|
| 7 |
+
"baseline_h16_rank_checkpoint_success": 0.29739130434782607,
|
| 8 |
+
"mean_success": 0.34840579710144925,
|
| 9 |
+
"std_success": 0.017764119937442965,
|
| 10 |
+
"mean_progress": 0.5669115647895419,
|
| 11 |
+
"mean_action_mse_to_best": 0.4032255988276523,
|
| 12 |
+
"gain_vs_h4": 0.051705797101449236,
|
| 13 |
+
"gain_vs_h16_rank_checkpoint": 0.051014492753623186,
|
| 14 |
+
"rows": [
|
| 15 |
+
{
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_knn2_scale0p50_safe_types_margin0p20.json",
|
| 18 |
+
"num_groups": 575,
|
| 19 |
+
"selection_mode": "retrieval_residual",
|
| 20 |
+
"num_candidates": 32,
|
| 21 |
+
"candidate_sigma": 0.0,
|
| 22 |
+
"selection_margin": 0.2,
|
| 23 |
+
"field_optim_steps": 0,
|
| 24 |
+
"field_optim_step_size": 0.0,
|
| 25 |
+
"field_optim_trust_radius": 0.0,
|
| 26 |
+
"field_optim_l2_penalty": 0.0,
|
| 27 |
+
"retrieval_neighbors": 2,
|
| 28 |
+
"retrieval_metric": "raw",
|
| 29 |
+
"retrieval_type_min_success": 0.0,
|
| 30 |
+
"retrieval_residual_scale": 0.5,
|
| 31 |
+
"retrieval_residual_anchor": "expert",
|
| 32 |
+
"policy_rollout_success_rate": 0.3356521739130435,
|
| 33 |
+
"policy_rollout_progress": 0.5535960750407337,
|
| 34 |
+
"oracle_success_rate": 0.8573913043478261,
|
| 35 |
+
"action_mse_to_best": 0.39054552934658915,
|
| 36 |
+
"best_policy_val": {
|
| 37 |
+
"bc_loss": 0.13721593966086706,
|
| 38 |
+
"field_effect_loss": 0.009290305380192068,
|
| 39 |
+
"field_potential_loss": 0.2666468388504452,
|
| 40 |
+
"field_preference_loss": 0.5130573478009965,
|
| 41 |
+
"lattice_edges": 3833.3333333333335,
|
| 42 |
+
"progress_mae": 0.1933159919248687,
|
| 43 |
+
"rank_acc": 0.8265031774838766,
|
| 44 |
+
"rank_loss": 0.5130523675017886,
|
| 45 |
+
"regret_mae": 0.3756548762321472,
|
| 46 |
+
"success_accuracy": 0.8773836526605818,
|
| 47 |
+
"total_loss": 1.5581054819954767
|
| 48 |
+
},
|
| 49 |
+
"per_task": {
|
| 50 |
+
"LiftPegUpright-v1": {
|
| 51 |
+
"action_mse_to_best": 0.31969772514973566,
|
| 52 |
+
"expert_success_rate": 0.8865979381443299,
|
| 53 |
+
"num_groups": 97,
|
| 54 |
+
"oracle_success_rate": 0.9278350515463918,
|
| 55 |
+
"policy_expert_regret": 1.0792337135248578,
|
| 56 |
+
"policy_oracle_regret": 1.1056491912333006,
|
| 57 |
+
"policy_rollout_progress": 0.570563180084081,
|
| 58 |
+
"policy_rollout_success_rate": 0.2268041237113402,
|
| 59 |
+
"restore_max_error": 4.76837158203125e-07
|
| 60 |
+
},
|
| 61 |
+
"PickCube-v1": {
|
| 62 |
+
"action_mse_to_best": 0.30456935832850063,
|
| 63 |
+
"expert_success_rate": 0.9375,
|
| 64 |
+
"num_groups": 208,
|
| 65 |
+
"oracle_success_rate": 0.9471153846153846,
|
| 66 |
+
"policy_expert_regret": 0.961126332063801,
|
| 67 |
+
"policy_oracle_regret": 0.9745491994903065,
|
| 68 |
+
"policy_rollout_progress": 0.6161403117629771,
|
| 69 |
+
"policy_rollout_success_rate": 0.34134615384615385,
|
| 70 |
+
"restore_max_error": 4.76837158203125e-07
|
| 71 |
+
},
|
| 72 |
+
"PullCube-v1": {
|
| 73 |
+
"action_mse_to_best": 0.606543309696309,
|
| 74 |
+
"expert_success_rate": 0.19480519480519481,
|
| 75 |
+
"num_groups": 77,
|
| 76 |
+
"oracle_success_rate": 0.36363636363636365,
|
| 77 |
+
"policy_expert_regret": 0.19823663703819083,
|
| 78 |
+
"policy_oracle_regret": 0.415207061242288,
|
| 79 |
+
"policy_rollout_progress": 0.3242795909059464,
|
| 80 |
+
"policy_rollout_success_rate": 0.22077922077922077,
|
| 81 |
+
"restore_max_error": 4.76837158203125e-07
|
| 82 |
+
},
|
| 83 |
+
"PushCube-v1": {
|
| 84 |
+
"action_mse_to_best": 0.366488233882379,
|
| 85 |
+
"expert_success_rate": 0.8617021276595744,
|
| 86 |
+
"num_groups": 94,
|
| 87 |
+
"oracle_success_rate": 0.9893617021276596,
|
| 88 |
+
"policy_expert_regret": 0.5719942746961371,
|
| 89 |
+
"policy_oracle_regret": 0.6495450359392674,
|
| 90 |
+
"policy_rollout_progress": 0.6908804959756263,
|
| 91 |
+
"policy_rollout_success_rate": 0.6595744680851063,
|
| 92 |
+
"restore_max_error": 4.76837158203125e-07
|
| 93 |
+
},
|
| 94 |
+
"StackCube-v1": {
|
| 95 |
+
"action_mse_to_best": 0.49544287546340265,
|
| 96 |
+
"expert_success_rate": 0.6767676767676768,
|
| 97 |
+
"num_groups": 99,
|
| 98 |
+
"oracle_success_rate": 0.8585858585858586,
|
| 99 |
+
"policy_expert_regret": 0.9810510797030998,
|
| 100 |
+
"policy_oracle_regret": 1.1552164670493867,
|
| 101 |
+
"policy_rollout_progress": 0.4535720677989902,
|
| 102 |
+
"policy_rollout_success_rate": 0.21212121212121213,
|
| 103 |
+
"restore_max_error": 3.948807716369629e-07
|
| 104 |
+
}
|
| 105 |
+
}
|
| 106 |
+
},
|
| 107 |
+
{
|
| 108 |
+
"seed": 1,
|
| 109 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_knn2_scale0p50_safe_types_margin0p20.json",
|
| 110 |
+
"num_groups": 575,
|
| 111 |
+
"selection_mode": "retrieval_residual",
|
| 112 |
+
"num_candidates": 32,
|
| 113 |
+
"candidate_sigma": 0.0,
|
| 114 |
+
"selection_margin": 0.2,
|
| 115 |
+
"field_optim_steps": 0,
|
| 116 |
+
"field_optim_step_size": 0.0,
|
| 117 |
+
"field_optim_trust_radius": 0.0,
|
| 118 |
+
"field_optim_l2_penalty": 0.0,
|
| 119 |
+
"retrieval_neighbors": 2,
|
| 120 |
+
"retrieval_metric": "raw",
|
| 121 |
+
"retrieval_type_min_success": 0.0,
|
| 122 |
+
"retrieval_residual_scale": 0.5,
|
| 123 |
+
"retrieval_residual_anchor": "expert",
|
| 124 |
+
"policy_rollout_success_rate": 0.3408695652173913,
|
| 125 |
+
"policy_rollout_progress": 0.5653851762673129,
|
| 126 |
+
"oracle_success_rate": 0.8695652173913043,
|
| 127 |
+
"action_mse_to_best": 0.3949822947933622,
|
| 128 |
+
"best_policy_val": {
|
| 129 |
+
"bc_loss": 0.1315989200439718,
|
| 130 |
+
"field_effect_loss": 0.010110370814800262,
|
| 131 |
+
"field_potential_loss": 0.31546536915832096,
|
| 132 |
+
"field_preference_loss": 0.4960637440284093,
|
| 133 |
+
"lattice_edges": 3833.3333333333335,
|
| 134 |
+
"progress_mae": 0.19023226698239645,
|
| 135 |
+
"rank_acc": 0.8255469501018524,
|
| 136 |
+
"rank_loss": 0.4960531195004781,
|
| 137 |
+
"regret_mae": 0.42156532737943864,
|
| 138 |
+
"success_accuracy": 0.8807473679383596,
|
| 139 |
+
"total_loss": 1.561984618504842
|
| 140 |
+
},
|
| 141 |
+
"per_task": {
|
| 142 |
+
"LiftPegUpright-v1": {
|
| 143 |
+
"action_mse_to_best": 0.32557108186482064,
|
| 144 |
+
"expert_success_rate": 0.8584070796460177,
|
| 145 |
+
"num_groups": 113,
|
| 146 |
+
"oracle_success_rate": 0.9380530973451328,
|
| 147 |
+
"policy_expert_regret": 0.987694250535121,
|
| 148 |
+
"policy_oracle_regret": 1.066896405631462,
|
| 149 |
+
"policy_rollout_progress": 0.6352153201546289,
|
| 150 |
+
"policy_rollout_success_rate": 0.23008849557522124,
|
| 151 |
+
"restore_max_error": 4.76837158203125e-07
|
| 152 |
+
},
|
| 153 |
+
"PickCube-v1": {
|
| 154 |
+
"action_mse_to_best": 0.3201394447368448,
|
| 155 |
+
"expert_success_rate": 0.9402173913043478,
|
| 156 |
+
"num_groups": 184,
|
| 157 |
+
"oracle_success_rate": 0.9456521739130435,
|
| 158 |
+
"policy_expert_regret": 1.0789333177710194,
|
| 159 |
+
"policy_oracle_regret": 1.0789573222459496,
|
| 160 |
+
"policy_rollout_progress": 0.5717033140624509,
|
| 161 |
+
"policy_rollout_success_rate": 0.28804347826086957,
|
| 162 |
+
"restore_max_error": 4.76837158203125e-07
|
| 163 |
+
},
|
| 164 |
+
"PullCube-v1": {
|
| 165 |
+
"action_mse_to_best": 0.5890702307224274,
|
| 166 |
+
"expert_success_rate": 0.25,
|
| 167 |
+
"num_groups": 76,
|
| 168 |
+
"oracle_success_rate": 0.40789473684210525,
|
| 169 |
+
"policy_expert_regret": 0.3023094067497081,
|
| 170 |
+
"policy_oracle_regret": 0.5491780981078351,
|
| 171 |
+
"policy_rollout_progress": 0.2765764820291416,
|
| 172 |
+
"policy_rollout_success_rate": 0.17105263157894737,
|
| 173 |
+
"restore_max_error": 4.76837158203125e-07
|
| 174 |
+
},
|
| 175 |
+
"PushCube-v1": {
|
| 176 |
+
"action_mse_to_best": 0.3623764889801408,
|
| 177 |
+
"expert_success_rate": 0.8198198198198198,
|
| 178 |
+
"num_groups": 111,
|
| 179 |
+
"oracle_success_rate": 1.0,
|
| 180 |
+
"policy_expert_regret": 0.33009791642695935,
|
| 181 |
+
"policy_oracle_regret": 0.36357842345495484,
|
| 182 |
+
"policy_rollout_progress": 0.8256107657342344,
|
| 183 |
+
"policy_rollout_success_rate": 0.8108108108108109,
|
| 184 |
+
"restore_max_error": 4.76837158203125e-07
|
| 185 |
+
},
|
| 186 |
+
"StackCube-v1": {
|
| 187 |
+
"action_mse_to_best": 0.5101813363975712,
|
| 188 |
+
"expert_success_rate": 0.6923076923076923,
|
| 189 |
+
"num_groups": 91,
|
| 190 |
+
"oracle_success_rate": 0.8571428571428571,
|
| 191 |
+
"policy_expert_regret": 1.1287748504143496,
|
| 192 |
+
"policy_oracle_regret": 1.3009226045438222,
|
| 193 |
+
"policy_rollout_progress": 0.38968272261567166,
|
| 194 |
+
"policy_rollout_success_rate": 0.15384615384615385,
|
| 195 |
+
"restore_max_error": 3.948807716369629e-07
|
| 196 |
+
}
|
| 197 |
+
}
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"seed": 2,
|
| 201 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_knn2_scale0p50_safe_types_margin0p20.json",
|
| 202 |
+
"num_groups": 575,
|
| 203 |
+
"selection_mode": "retrieval_residual",
|
| 204 |
+
"num_candidates": 32,
|
| 205 |
+
"candidate_sigma": 0.0,
|
| 206 |
+
"selection_margin": 0.2,
|
| 207 |
+
"field_optim_steps": 0,
|
| 208 |
+
"field_optim_step_size": 0.0,
|
| 209 |
+
"field_optim_trust_radius": 0.0,
|
| 210 |
+
"field_optim_l2_penalty": 0.0,
|
| 211 |
+
"retrieval_neighbors": 2,
|
| 212 |
+
"retrieval_metric": "raw",
|
| 213 |
+
"retrieval_type_min_success": 0.0,
|
| 214 |
+
"retrieval_residual_scale": 0.5,
|
| 215 |
+
"retrieval_residual_anchor": "expert",
|
| 216 |
+
"policy_rollout_success_rate": 0.36869565217391304,
|
| 217 |
+
"policy_rollout_progress": 0.5817534430605789,
|
| 218 |
+
"oracle_success_rate": 0.8765217391304347,
|
| 219 |
+
"action_mse_to_best": 0.4241489723430056,
|
| 220 |
+
"best_policy_val": {
|
| 221 |
+
"bc_loss": 0.11367896075050037,
|
| 222 |
+
"field_effect_loss": 0.009670218582161598,
|
| 223 |
+
"field_potential_loss": 0.2641640139950646,
|
| 224 |
+
"field_preference_loss": 0.5130490180518892,
|
| 225 |
+
"lattice_edges": 3833.3333333333335,
|
| 226 |
+
"progress_mae": 0.2021729110015763,
|
| 227 |
+
"rank_acc": 0.8333857821093665,
|
| 228 |
+
"rank_loss": 0.5130119257503085,
|
| 229 |
+
"regret_mae": 0.3958987047274907,
|
| 230 |
+
"success_accuracy": 0.8680730561415354,
|
| 231 |
+
"total_loss": 1.4394984311527677
|
| 232 |
+
},
|
| 233 |
+
"per_task": {
|
| 234 |
+
"LiftPegUpright-v1": {
|
| 235 |
+
"action_mse_to_best": 0.3546536646366197,
|
| 236 |
+
"expert_success_rate": 0.8229166666666666,
|
| 237 |
+
"num_groups": 96,
|
| 238 |
+
"oracle_success_rate": 0.9270833333333334,
|
| 239 |
+
"policy_expert_regret": 0.8470402403424183,
|
| 240 |
+
"policy_oracle_regret": 0.9533149556567272,
|
| 241 |
+
"policy_rollout_progress": 0.6469602342694998,
|
| 242 |
+
"policy_rollout_success_rate": 0.3333333333333333,
|
| 243 |
+
"restore_max_error": 3.5762786865234375e-07
|
| 244 |
+
},
|
| 245 |
+
"PickCube-v1": {
|
| 246 |
+
"action_mse_to_best": 0.33931025653851754,
|
| 247 |
+
"expert_success_rate": 0.9444444444444444,
|
| 248 |
+
"num_groups": 198,
|
| 249 |
+
"oracle_success_rate": 0.9595959595959596,
|
| 250 |
+
"policy_expert_regret": 0.9768050375930739,
|
| 251 |
+
"policy_oracle_regret": 0.9838884881217823,
|
| 252 |
+
"policy_rollout_progress": 0.6325268320494679,
|
| 253 |
+
"policy_rollout_success_rate": 0.3383838383838384,
|
| 254 |
+
"restore_max_error": 4.76837158203125e-07
|
| 255 |
+
},
|
| 256 |
+
"PullCube-v1": {
|
| 257 |
+
"action_mse_to_best": 0.6406738382246759,
|
| 258 |
+
"expert_success_rate": 0.24444444444444444,
|
| 259 |
+
"num_groups": 90,
|
| 260 |
+
"oracle_success_rate": 0.4666666666666667,
|
| 261 |
+
"policy_expert_regret": 0.2732478534043872,
|
| 262 |
+
"policy_oracle_regret": 0.500017503573104,
|
| 263 |
+
"policy_rollout_progress": 0.3393847736150646,
|
| 264 |
+
"policy_rollout_success_rate": 0.23333333333333334,
|
| 265 |
+
"restore_max_error": 4.0978193283081055e-07
|
| 266 |
+
},
|
| 267 |
+
"PushCube-v1": {
|
| 268 |
+
"action_mse_to_best": 0.39339480984328995,
|
| 269 |
+
"expert_success_rate": 0.8514851485148515,
|
| 270 |
+
"num_groups": 101,
|
| 271 |
+
"oracle_success_rate": 1.0,
|
| 272 |
+
"policy_expert_regret": 0.39989533297496266,
|
| 273 |
+
"policy_oracle_regret": 0.43690602510872456,
|
| 274 |
+
"policy_rollout_progress": 0.7908167471685031,
|
| 275 |
+
"policy_rollout_success_rate": 0.7722772277227723,
|
| 276 |
+
"restore_max_error": 4.76837158203125e-07
|
| 277 |
+
},
|
| 278 |
+
"StackCube-v1": {
|
| 279 |
+
"action_mse_to_best": 0.502910614034368,
|
| 280 |
+
"expert_success_rate": 0.7666666666666667,
|
| 281 |
+
"num_groups": 90,
|
| 282 |
+
"oracle_success_rate": 0.9111111111111111,
|
| 283 |
+
"policy_expert_regret": 1.1562208301491208,
|
| 284 |
+
"policy_oracle_regret": 1.3182081157962482,
|
| 285 |
+
"policy_rollout_progress": 0.40825126038657295,
|
| 286 |
+
"policy_rollout_success_rate": 0.15555555555555556,
|
| 287 |
+
"restore_max_error": 4.76837158203125e-07
|
| 288 |
+
}
|
| 289 |
+
}
|
| 290 |
+
}
|
| 291 |
+
]
|
| 292 |
+
}
|
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p50_safe_types_margin0p20_summary.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# h=16 Best-Policy Checkpoint Rollout
|
| 2 |
+
|
| 3 |
+
Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
|
| 4 |
+
Objective: `near_miss_policy_bc5`
|
| 5 |
+
Result file: `policy_rollout_retrieval_residual_knn2_scale0p50_safe_types_margin0p20.json`
|
| 6 |
+
Completed seeds: 3
|
| 7 |
+
Baseline h=4 policy success: 29.67%
|
| 8 |
+
Baseline h=16 rank-checkpoint success: 29.74%
|
| 9 |
+
|
| 10 |
+
Mean success: 34.84% +/- 1.78%
|
| 11 |
+
Gain vs h=16 rank checkpoint: +5.10%
|
| 12 |
+
Mean progress: 56.69%
|
| 13 |
+
Mean action MSE to best: 0.403
|
| 14 |
+
|
| 15 |
+
| seed | mode | k | retrieval K | retrieval metric | residual anchor | min type success | residual scale | margin | sigma | opt steps | trust | success | progress | oracle | action MSE |
|
| 16 |
+
|---:|---|---:|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
| 17 |
+
| 0 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.50 | 0.200 | 0.00 | 0 | 0.00 | 33.57% | 55.36% | 85.74% | 0.391 |
|
| 18 |
+
| 1 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.50 | 0.200 | 0.00 | 0 | 0.00 | 34.09% | 56.54% | 86.96% | 0.395 |
|
| 19 |
+
| 2 | retrieval_residual | 32 | 2 | raw | expert | 0.00 | 0.50 | 0.200 | 0.00 | 0 | 0.00 | 36.87% | 58.18% | 87.65% | 0.424 |
|
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn4_scale0p35_safe_types_margin0p20_summary.json
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
|
| 3 |
+
"objective": "near_miss_policy_bc5",
|
| 4 |
+
"out_name": "policy_rollout_retrieval_residual_knn4_scale0p35_safe_types_margin0p20.json",
|
| 5 |
+
"num_completed": 3,
|
| 6 |
+
"baseline_h4_policy_success": 0.2967,
|
| 7 |
+
"baseline_h16_rank_checkpoint_success": 0.29739130434782607,
|
| 8 |
+
"mean_success": 0.34898550724637684,
|
| 9 |
+
"std_success": 0.018595959341849794,
|
| 10 |
+
"mean_progress": 0.5656977585057061,
|
| 11 |
+
"mean_action_mse_to_best": 0.39866841265310843,
|
| 12 |
+
"gain_vs_h4": 0.05228550724637682,
|
| 13 |
+
"gain_vs_h16_rank_checkpoint": 0.05159420289855077,
|
| 14 |
+
"rows": [
|
| 15 |
+
{
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_knn4_scale0p35_safe_types_margin0p20.json",
|
| 18 |
+
"num_groups": 575,
|
| 19 |
+
"selection_mode": "retrieval_residual",
|
| 20 |
+
"num_candidates": 64,
|
| 21 |
+
"candidate_sigma": 0.0,
|
| 22 |
+
"selection_margin": 0.2,
|
| 23 |
+
"field_optim_steps": 0,
|
| 24 |
+
"field_optim_step_size": 0.0,
|
| 25 |
+
"field_optim_trust_radius": 0.0,
|
| 26 |
+
"field_optim_l2_penalty": 0.0,
|
| 27 |
+
"retrieval_neighbors": 4,
|
| 28 |
+
"retrieval_metric": "raw",
|
| 29 |
+
"retrieval_type_min_success": 0.0,
|
| 30 |
+
"retrieval_residual_scale": 0.35,
|
| 31 |
+
"retrieval_residual_anchor": "expert",
|
| 32 |
+
"policy_rollout_success_rate": 0.3373913043478261,
|
| 33 |
+
"policy_rollout_progress": 0.5522649569777043,
|
| 34 |
+
"oracle_success_rate": 0.8573913043478261,
|
| 35 |
+
"action_mse_to_best": 0.3842362688878632,
|
| 36 |
+
"best_policy_val": {
|
| 37 |
+
"bc_loss": 0.13721593966086706,
|
| 38 |
+
"field_effect_loss": 0.009290305380192068,
|
| 39 |
+
"field_potential_loss": 0.2666468388504452,
|
| 40 |
+
"field_preference_loss": 0.5130573478009965,
|
| 41 |
+
"lattice_edges": 3833.3333333333335,
|
| 42 |
+
"progress_mae": 0.1933159919248687,
|
| 43 |
+
"rank_acc": 0.8265031774838766,
|
| 44 |
+
"rank_loss": 0.5130523675017886,
|
| 45 |
+
"regret_mae": 0.3756548762321472,
|
| 46 |
+
"success_accuracy": 0.8773836526605818,
|
| 47 |
+
"total_loss": 1.5581054819954767
|
| 48 |
+
},
|
| 49 |
+
"per_task": {
|
| 50 |
+
"LiftPegUpright-v1": {
|
| 51 |
+
"action_mse_to_best": 0.32106486344517965,
|
| 52 |
+
"expert_success_rate": 0.8865979381443299,
|
| 53 |
+
"num_groups": 97,
|
| 54 |
+
"oracle_success_rate": 0.9278350515463918,
|
| 55 |
+
"policy_expert_regret": 1.0924889440696264,
|
| 56 |
+
"policy_oracle_regret": 1.1189044217780693,
|
| 57 |
+
"policy_rollout_progress": 0.5676172278898278,
|
| 58 |
+
"policy_rollout_success_rate": 0.21649484536082475,
|
| 59 |
+
"restore_max_error": 4.76837158203125e-07
|
| 60 |
+
},
|
| 61 |
+
"PickCube-v1": {
|
| 62 |
+
"action_mse_to_best": 0.3029568468650373,
|
| 63 |
+
"expert_success_rate": 0.9375,
|
| 64 |
+
"num_groups": 208,
|
| 65 |
+
"oracle_success_rate": 0.9471153846153846,
|
| 66 |
+
"policy_expert_regret": 0.9772481287184817,
|
| 67 |
+
"policy_oracle_regret": 0.9906709961449871,
|
| 68 |
+
"policy_rollout_progress": 0.6096338997236811,
|
| 69 |
+
"policy_rollout_success_rate": 0.3317307692307692,
|
| 70 |
+
"restore_max_error": 4.76837158203125e-07
|
| 71 |
+
},
|
| 72 |
+
"PullCube-v1": {
|
| 73 |
+
"action_mse_to_best": 0.6057495275488147,
|
| 74 |
+
"expert_success_rate": 0.19480519480519481,
|
| 75 |
+
"num_groups": 77,
|
| 76 |
+
"oracle_success_rate": 0.36363636363636365,
|
| 77 |
+
"policy_expert_regret": 0.19458804545657976,
|
| 78 |
+
"policy_oracle_regret": 0.3415201688306285,
|
| 79 |
+
"policy_rollout_progress": 0.35899942796435447,
|
| 80 |
+
"policy_rollout_success_rate": 0.2597402597402597,
|
| 81 |
+
"restore_max_error": 4.76837158203125e-07
|
| 82 |
+
},
|
| 83 |
+
"PushCube-v1": {
|
| 84 |
+
"action_mse_to_best": 0.36560009463511883,
|
| 85 |
+
"expert_success_rate": 0.8617021276595744,
|
| 86 |
+
"num_groups": 94,
|
| 87 |
+
"oracle_success_rate": 0.9893617021276596,
|
| 88 |
+
"policy_expert_regret": 0.5719942746961371,
|
| 89 |
+
"policy_oracle_regret": 0.6683584781403237,
|
| 90 |
+
"policy_rollout_progress": 0.6827053516469104,
|
| 91 |
+
"policy_rollout_success_rate": 0.648936170212766,
|
| 92 |
+
"restore_max_error": 4.76837158203125e-07
|
| 93 |
+
},
|
| 94 |
+
"StackCube-v1": {
|
| 95 |
+
"action_mse_to_best": 0.46230723425708337,
|
| 96 |
+
"expert_success_rate": 0.6767676767676768,
|
| 97 |
+
"num_groups": 99,
|
| 98 |
+
"oracle_success_rate": 0.8585858585858586,
|
| 99 |
+
"policy_expert_regret": 0.9712549740015858,
|
| 100 |
+
"policy_oracle_regret": 1.145431272158719,
|
| 101 |
+
"policy_rollout_progress": 0.44315524248763766,
|
| 102 |
+
"policy_rollout_success_rate": 0.23232323232323232,
|
| 103 |
+
"restore_max_error": 3.948807716369629e-07
|
| 104 |
+
}
|
| 105 |
+
}
|
| 106 |
+
},
|
| 107 |
+
{
|
| 108 |
+
"seed": 1,
|
| 109 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_knn4_scale0p35_safe_types_margin0p20.json",
|
| 110 |
+
"num_groups": 575,
|
| 111 |
+
"selection_mode": "retrieval_residual",
|
| 112 |
+
"num_candidates": 64,
|
| 113 |
+
"candidate_sigma": 0.0,
|
| 114 |
+
"selection_margin": 0.2,
|
| 115 |
+
"field_optim_steps": 0,
|
| 116 |
+
"field_optim_step_size": 0.0,
|
| 117 |
+
"field_optim_trust_radius": 0.0,
|
| 118 |
+
"field_optim_l2_penalty": 0.0,
|
| 119 |
+
"retrieval_neighbors": 4,
|
| 120 |
+
"retrieval_metric": "raw",
|
| 121 |
+
"retrieval_type_min_success": 0.0,
|
| 122 |
+
"retrieval_residual_scale": 0.35,
|
| 123 |
+
"retrieval_residual_anchor": "expert",
|
| 124 |
+
"policy_rollout_success_rate": 0.3391304347826087,
|
| 125 |
+
"policy_rollout_progress": 0.5635609641363439,
|
| 126 |
+
"oracle_success_rate": 0.8695652173913043,
|
| 127 |
+
"action_mse_to_best": 0.39139225294732527,
|
| 128 |
+
"best_policy_val": {
|
| 129 |
+
"bc_loss": 0.1315989200439718,
|
| 130 |
+
"field_effect_loss": 0.010110370814800262,
|
| 131 |
+
"field_potential_loss": 0.31546536915832096,
|
| 132 |
+
"field_preference_loss": 0.4960637440284093,
|
| 133 |
+
"lattice_edges": 3833.3333333333335,
|
| 134 |
+
"progress_mae": 0.19023226698239645,
|
| 135 |
+
"rank_acc": 0.8255469501018524,
|
| 136 |
+
"rank_loss": 0.4960531195004781,
|
| 137 |
+
"regret_mae": 0.42156532737943864,
|
| 138 |
+
"success_accuracy": 0.8807473679383596,
|
| 139 |
+
"total_loss": 1.561984618504842
|
| 140 |
+
},
|
| 141 |
+
"per_task": {
|
| 142 |
+
"LiftPegUpright-v1": {
|
| 143 |
+
"action_mse_to_best": 0.32611669615669325,
|
| 144 |
+
"expert_success_rate": 0.8584070796460177,
|
| 145 |
+
"num_groups": 113,
|
| 146 |
+
"oracle_success_rate": 0.9380530973451328,
|
| 147 |
+
"policy_expert_regret": 0.987694250535121,
|
| 148 |
+
"policy_oracle_regret": 1.066896405631462,
|
| 149 |
+
"policy_rollout_progress": 0.6352153201546289,
|
| 150 |
+
"policy_rollout_success_rate": 0.23008849557522124,
|
| 151 |
+
"restore_max_error": 4.76837158203125e-07
|
| 152 |
+
},
|
| 153 |
+
"PickCube-v1": {
|
| 154 |
+
"action_mse_to_best": 0.3206069803310801,
|
| 155 |
+
"expert_success_rate": 0.9402173913043478,
|
| 156 |
+
"num_groups": 184,
|
| 157 |
+
"oracle_success_rate": 0.9456521739130435,
|
| 158 |
+
"policy_expert_regret": 1.0710192661702067,
|
| 159 |
+
"policy_oracle_regret": 1.0710432706451367,
|
| 160 |
+
"policy_rollout_progress": 0.5741825830545682,
|
| 161 |
+
"policy_rollout_success_rate": 0.29347826086956524,
|
| 162 |
+
"restore_max_error": 4.76837158203125e-07
|
| 163 |
+
},
|
| 164 |
+
"PullCube-v1": {
|
| 165 |
+
"action_mse_to_best": 0.5791849780239557,
|
| 166 |
+
"expert_success_rate": 0.25,
|
| 167 |
+
"num_groups": 76,
|
| 168 |
+
"oracle_success_rate": 0.40789473684210525,
|
| 169 |
+
"policy_expert_regret": 0.301088224434735,
|
| 170 |
+
"policy_oracle_regret": 0.5471240192112562,
|
| 171 |
+
"policy_rollout_progress": 0.2791815299883877,
|
| 172 |
+
"policy_rollout_success_rate": 0.17105263157894737,
|
| 173 |
+
"restore_max_error": 4.76837158203125e-07
|
| 174 |
+
},
|
| 175 |
+
"PushCube-v1": {
|
| 176 |
+
"action_mse_to_best": 0.35639740489758887,
|
| 177 |
+
"expert_success_rate": 0.8198198198198198,
|
| 178 |
+
"num_groups": 111,
|
| 179 |
+
"oracle_success_rate": 1.0,
|
| 180 |
+
"policy_expert_regret": 0.3476062170020095,
|
| 181 |
+
"policy_oracle_regret": 0.3980809340068886,
|
| 182 |
+
"policy_rollout_progress": 0.8091262732003186,
|
| 183 |
+
"policy_rollout_success_rate": 0.7927927927927928,
|
| 184 |
+
"restore_max_error": 4.76837158203125e-07
|
| 185 |
+
},
|
| 186 |
+
"StackCube-v1": {
|
| 187 |
+
"action_mse_to_best": 0.5014231222487249,
|
| 188 |
+
"expert_success_rate": 0.6923076923076923,
|
| 189 |
+
"num_groups": 91,
|
| 190 |
+
"oracle_success_rate": 0.8571428571428571,
|
| 191 |
+
"policy_expert_regret": 1.127382679463743,
|
| 192 |
+
"policy_oracle_regret": 1.2995304335932156,
|
| 193 |
+
"policy_rollout_progress": 0.39107489356627834,
|
| 194 |
+
"policy_rollout_success_rate": 0.15384615384615385,
|
| 195 |
+
"restore_max_error": 3.948807716369629e-07
|
| 196 |
+
}
|
| 197 |
+
}
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"seed": 2,
|
| 201 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_knn4_scale0p35_safe_types_margin0p20.json",
|
| 202 |
+
"num_groups": 575,
|
| 203 |
+
"selection_mode": "retrieval_residual",
|
| 204 |
+
"num_candidates": 64,
|
| 205 |
+
"candidate_sigma": 0.0,
|
| 206 |
+
"selection_margin": 0.2,
|
| 207 |
+
"field_optim_steps": 0,
|
| 208 |
+
"field_optim_step_size": 0.0,
|
| 209 |
+
"field_optim_trust_radius": 0.0,
|
| 210 |
+
"field_optim_l2_penalty": 0.0,
|
| 211 |
+
"retrieval_neighbors": 4,
|
| 212 |
+
"retrieval_metric": "raw",
|
| 213 |
+
"retrieval_type_min_success": 0.0,
|
| 214 |
+
"retrieval_residual_scale": 0.35,
|
| 215 |
+
"retrieval_residual_anchor": "expert",
|
| 216 |
+
"policy_rollout_success_rate": 0.37043478260869567,
|
| 217 |
+
"policy_rollout_progress": 0.5812673544030701,
|
| 218 |
+
"oracle_success_rate": 0.8765217391304347,
|
| 219 |
+
"action_mse_to_best": 0.4203767161241368,
|
| 220 |
+
"best_policy_val": {
|
| 221 |
+
"bc_loss": 0.11367896075050037,
|
| 222 |
+
"field_effect_loss": 0.009670218582161598,
|
| 223 |
+
"field_potential_loss": 0.2641640139950646,
|
| 224 |
+
"field_preference_loss": 0.5130490180518892,
|
| 225 |
+
"lattice_edges": 3833.3333333333335,
|
| 226 |
+
"progress_mae": 0.2021729110015763,
|
| 227 |
+
"rank_acc": 0.8333857821093665,
|
| 228 |
+
"rank_loss": 0.5130119257503085,
|
| 229 |
+
"regret_mae": 0.3958987047274907,
|
| 230 |
+
"success_accuracy": 0.8680730561415354,
|
| 231 |
+
"total_loss": 1.4394984311527677
|
| 232 |
+
},
|
| 233 |
+
"per_task": {
|
| 234 |
+
"LiftPegUpright-v1": {
|
| 235 |
+
"action_mse_to_best": 0.35413535257976037,
|
| 236 |
+
"expert_success_rate": 0.8229166666666666,
|
| 237 |
+
"num_groups": 96,
|
| 238 |
+
"oracle_success_rate": 0.9270833333333334,
|
| 239 |
+
"policy_expert_regret": 0.8470402403424183,
|
| 240 |
+
"policy_oracle_regret": 0.9533149556567272,
|
| 241 |
+
"policy_rollout_progress": 0.6469602342694998,
|
| 242 |
+
"policy_rollout_success_rate": 0.3333333333333333,
|
| 243 |
+
"restore_max_error": 3.5762786865234375e-07
|
| 244 |
+
},
|
| 245 |
+
"PickCube-v1": {
|
| 246 |
+
"action_mse_to_best": 0.3371933022269396,
|
| 247 |
+
"expert_success_rate": 0.9444444444444444,
|
| 248 |
+
"num_groups": 198,
|
| 249 |
+
"oracle_success_rate": 0.9595959595959596,
|
| 250 |
+
"policy_expert_regret": 0.9763947898954756,
|
| 251 |
+
"policy_oracle_regret": 0.9834782404241839,
|
| 252 |
+
"policy_rollout_progress": 0.6329370797470664,
|
| 253 |
+
"policy_rollout_success_rate": 0.3383838383838384,
|
| 254 |
+
"restore_max_error": 4.76837158203125e-07
|
| 255 |
+
},
|
| 256 |
+
"PullCube-v1": {
|
| 257 |
+
"action_mse_to_best": 0.643382321132554,
|
| 258 |
+
"expert_success_rate": 0.24444444444444444,
|
| 259 |
+
"num_groups": 90,
|
| 260 |
+
"oracle_success_rate": 0.4666666666666667,
|
| 261 |
+
"policy_expert_regret": 0.27503268005394704,
|
| 262 |
+
"policy_oracle_regret": 0.4827883422443645,
|
| 263 |
+
"policy_rollout_progress": 0.34550282383269304,
|
| 264 |
+
"policy_rollout_success_rate": 0.24444444444444444,
|
| 265 |
+
"restore_max_error": 4.0978193283081055e-07
|
| 266 |
+
},
|
| 267 |
+
"PushCube-v1": {
|
| 268 |
+
"action_mse_to_best": 0.3863796532006547,
|
| 269 |
+
"expert_success_rate": 0.8514851485148515,
|
| 270 |
+
"num_groups": 101,
|
| 271 |
+
"oracle_success_rate": 1.0,
|
| 272 |
+
"policy_expert_regret": 0.381283899787629,
|
| 273 |
+
"policy_oracle_regret": 0.4539335219576807,
|
| 274 |
+
"policy_rollout_progress": 0.7836902404185568,
|
| 275 |
+
"policy_rollout_success_rate": 0.7623762376237624,
|
| 276 |
+
"restore_max_error": 4.76837158203125e-07
|
| 277 |
+
},
|
| 278 |
+
"StackCube-v1": {
|
| 279 |
+
"action_mse_to_best": 0.48918433563990726,
|
| 280 |
+
"expert_success_rate": 0.7666666666666667,
|
| 281 |
+
"num_groups": 90,
|
| 282 |
+
"oracle_success_rate": 0.9111111111111111,
|
| 283 |
+
"policy_expert_regret": 1.160153476893902,
|
| 284 |
+
"policy_oracle_regret": 1.309225642018848,
|
| 285 |
+
"policy_rollout_progress": 0.40612262305286195,
|
| 286 |
+
"policy_rollout_success_rate": 0.16666666666666666,
|
| 287 |
+
"restore_max_error": 4.76837158203125e-07
|
| 288 |
+
}
|
| 289 |
+
}
|
| 290 |
+
}
|
| 291 |
+
]
|
| 292 |
+
}
|
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn4_scale0p35_safe_types_margin0p20_summary.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# h=16 Best-Policy Checkpoint Rollout
|
| 2 |
+
|
| 3 |
+
Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
|
| 4 |
+
Objective: `near_miss_policy_bc5`
|
| 5 |
+
Result file: `policy_rollout_retrieval_residual_knn4_scale0p35_safe_types_margin0p20.json`
|
| 6 |
+
Completed seeds: 3
|
| 7 |
+
Baseline h=4 policy success: 29.67%
|
| 8 |
+
Baseline h=16 rank-checkpoint success: 29.74%
|
| 9 |
+
|
| 10 |
+
Mean success: 34.90% +/- 1.86%
|
| 11 |
+
Gain vs h=16 rank checkpoint: +5.16%
|
| 12 |
+
Mean progress: 56.57%
|
| 13 |
+
Mean action MSE to best: 0.399
|
| 14 |
+
|
| 15 |
+
| seed | mode | k | retrieval K | retrieval metric | residual anchor | min type success | residual scale | margin | sigma | opt steps | trust | success | progress | oracle | action MSE |
|
| 16 |
+
|---:|---|---:|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
| 17 |
+
| 0 | retrieval_residual | 64 | 4 | raw | expert | 0.00 | 0.35 | 0.200 | 0.00 | 0 | 0.00 | 33.74% | 55.23% | 85.74% | 0.384 |
|
| 18 |
+
| 1 | retrieval_residual | 64 | 4 | raw | expert | 0.00 | 0.35 | 0.200 | 0.00 | 0 | 0.00 | 33.91% | 56.36% | 86.96% | 0.391 |
|
| 19 |
+
| 2 | retrieval_residual | 64 | 4 | raw | expert | 0.00 | 0.35 | 0.200 | 0.00 | 0 | 0.00 | 37.04% | 58.13% | 87.65% | 0.420 |
|
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn8_scale0p35_safe_types_margin0p20_summary.json
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
|
| 3 |
+
"objective": "near_miss_policy_bc5",
|
| 4 |
+
"out_name": "policy_rollout_retrieval_residual_knn8_scale0p35_safe_types_margin0p20.json",
|
| 5 |
+
"num_completed": 3,
|
| 6 |
+
"baseline_h4_policy_success": 0.2967,
|
| 7 |
+
"baseline_h16_rank_checkpoint_success": 0.29739130434782607,
|
| 8 |
+
"mean_success": 0.34840579710144925,
|
| 9 |
+
"std_success": 0.015194599909659314,
|
| 10 |
+
"mean_progress": 0.5652518557331074,
|
| 11 |
+
"mean_action_mse_to_best": 0.40407356052349014,
|
| 12 |
+
"gain_vs_h4": 0.051705797101449236,
|
| 13 |
+
"gain_vs_h16_rank_checkpoint": 0.051014492753623186,
|
| 14 |
+
"rows": [
|
| 15 |
+
{
|
| 16 |
+
"seed": 0,
|
| 17 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_knn8_scale0p35_safe_types_margin0p20.json",
|
| 18 |
+
"num_groups": 575,
|
| 19 |
+
"selection_mode": "retrieval_residual",
|
| 20 |
+
"num_candidates": 128,
|
| 21 |
+
"candidate_sigma": 0.0,
|
| 22 |
+
"selection_margin": 0.2,
|
| 23 |
+
"field_optim_steps": 0,
|
| 24 |
+
"field_optim_step_size": 0.0,
|
| 25 |
+
"field_optim_trust_radius": 0.0,
|
| 26 |
+
"field_optim_l2_penalty": 0.0,
|
| 27 |
+
"retrieval_neighbors": 8,
|
| 28 |
+
"retrieval_metric": "raw",
|
| 29 |
+
"retrieval_type_min_success": 0.0,
|
| 30 |
+
"retrieval_residual_scale": 0.35,
|
| 31 |
+
"retrieval_residual_anchor": "expert",
|
| 32 |
+
"policy_rollout_success_rate": 0.3356521739130435,
|
| 33 |
+
"policy_rollout_progress": 0.5521184356652362,
|
| 34 |
+
"oracle_success_rate": 0.8573913043478261,
|
| 35 |
+
"action_mse_to_best": 0.38924177392102455,
|
| 36 |
+
"best_policy_val": {
|
| 37 |
+
"bc_loss": 0.13721593966086706,
|
| 38 |
+
"field_effect_loss": 0.009290305380192068,
|
| 39 |
+
"field_potential_loss": 0.2666468388504452,
|
| 40 |
+
"field_preference_loss": 0.5130573478009965,
|
| 41 |
+
"lattice_edges": 3833.3333333333335,
|
| 42 |
+
"progress_mae": 0.1933159919248687,
|
| 43 |
+
"rank_acc": 0.8265031774838766,
|
| 44 |
+
"rank_loss": 0.5130523675017886,
|
| 45 |
+
"regret_mae": 0.3756548762321472,
|
| 46 |
+
"success_accuracy": 0.8773836526605818,
|
| 47 |
+
"total_loss": 1.5581054819954767
|
| 48 |
+
},
|
| 49 |
+
"per_task": {
|
| 50 |
+
"LiftPegUpright-v1": {
|
| 51 |
+
"action_mse_to_best": 0.32106486344517965,
|
| 52 |
+
"expert_success_rate": 0.8865979381443299,
|
| 53 |
+
"num_groups": 97,
|
| 54 |
+
"oracle_success_rate": 0.9278350515463918,
|
| 55 |
+
"policy_expert_regret": 1.0924889440696264,
|
| 56 |
+
"policy_oracle_regret": 1.1189044217780693,
|
| 57 |
+
"policy_rollout_progress": 0.5676172278898278,
|
| 58 |
+
"policy_rollout_success_rate": 0.21649484536082475,
|
| 59 |
+
"restore_max_error": 4.76837158203125e-07
|
| 60 |
+
},
|
| 61 |
+
"PickCube-v1": {
|
| 62 |
+
"action_mse_to_best": 0.3034135256893933,
|
| 63 |
+
"expert_success_rate": 0.9375,
|
| 64 |
+
"num_groups": 208,
|
| 65 |
+
"oracle_success_rate": 0.9471153846153846,
|
| 66 |
+
"policy_expert_regret": 0.9704194357260488,
|
| 67 |
+
"policy_oracle_regret": 0.9838423031525543,
|
| 68 |
+
"policy_rollout_progress": 0.6116549004084215,
|
| 69 |
+
"policy_rollout_success_rate": 0.33653846153846156,
|
| 70 |
+
"restore_max_error": 4.76837158203125e-07
|
| 71 |
+
},
|
| 72 |
+
"PullCube-v1": {
|
| 73 |
+
"action_mse_to_best": 0.6136944288170183,
|
| 74 |
+
"expert_success_rate": 0.19480519480519481,
|
| 75 |
+
"num_groups": 77,
|
| 76 |
+
"oracle_success_rate": 0.36363636363636365,
|
| 77 |
+
"policy_expert_regret": 0.19560104213328428,
|
| 78 |
+
"policy_oracle_regret": 0.3662953276531892,
|
| 79 |
+
"policy_rollout_progress": 0.3472112821288068,
|
| 80 |
+
"policy_rollout_success_rate": 0.24675324675324675,
|
| 81 |
+
"restore_max_error": 4.76837158203125e-07
|
| 82 |
+
},
|
| 83 |
+
"PushCube-v1": {
|
| 84 |
+
"action_mse_to_best": 0.36909177403976307,
|
| 85 |
+
"expert_success_rate": 0.8617021276595744,
|
| 86 |
+
"num_groups": 94,
|
| 87 |
+
"oracle_success_rate": 0.9893617021276596,
|
| 88 |
+
"policy_expert_regret": 0.5719942746961371,
|
| 89 |
+
"policy_oracle_regret": 0.6495450359392674,
|
| 90 |
+
"policy_rollout_progress": 0.6908804959756263,
|
| 91 |
+
"policy_rollout_success_rate": 0.6595744680851063,
|
| 92 |
+
"restore_max_error": 4.76837158203125e-07
|
| 93 |
+
},
|
| 94 |
+
"StackCube-v1": {
|
| 95 |
+
"action_mse_to_best": 0.4809254255390378,
|
| 96 |
+
"expert_success_rate": 0.6767676767676768,
|
| 97 |
+
"num_groups": 99,
|
| 98 |
+
"oracle_success_rate": 0.8585858585858586,
|
| 99 |
+
"policy_expert_regret": 0.9951478450587301,
|
| 100 |
+
"policy_oracle_regret": 1.1693241432158634,
|
| 101 |
+
"policy_rollout_progress": 0.43946439163251355,
|
| 102 |
+
"policy_rollout_success_rate": 0.21212121212121213,
|
| 103 |
+
"restore_max_error": 3.948807716369629e-07
|
| 104 |
+
}
|
| 105 |
+
}
|
| 106 |
+
},
|
| 107 |
+
{
|
| 108 |
+
"seed": 1,
|
| 109 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_knn8_scale0p35_safe_types_margin0p20.json",
|
| 110 |
+
"num_groups": 575,
|
| 111 |
+
"selection_mode": "retrieval_residual",
|
| 112 |
+
"num_candidates": 128,
|
| 113 |
+
"candidate_sigma": 0.0,
|
| 114 |
+
"selection_margin": 0.2,
|
| 115 |
+
"field_optim_steps": 0,
|
| 116 |
+
"field_optim_step_size": 0.0,
|
| 117 |
+
"field_optim_trust_radius": 0.0,
|
| 118 |
+
"field_optim_l2_penalty": 0.0,
|
| 119 |
+
"retrieval_neighbors": 8,
|
| 120 |
+
"retrieval_metric": "raw",
|
| 121 |
+
"retrieval_type_min_success": 0.0,
|
| 122 |
+
"retrieval_residual_scale": 0.35,
|
| 123 |
+
"retrieval_residual_anchor": "expert",
|
| 124 |
+
"policy_rollout_success_rate": 0.3443478260869565,
|
| 125 |
+
"policy_rollout_progress": 0.5678077366737568,
|
| 126 |
+
"oracle_success_rate": 0.8695652173913043,
|
| 127 |
+
"action_mse_to_best": 0.3975022074547799,
|
| 128 |
+
"best_policy_val": {
|
| 129 |
+
"bc_loss": 0.1315989200439718,
|
| 130 |
+
"field_effect_loss": 0.010110370814800262,
|
| 131 |
+
"field_potential_loss": 0.31546536915832096,
|
| 132 |
+
"field_preference_loss": 0.4960637440284093,
|
| 133 |
+
"lattice_edges": 3833.3333333333335,
|
| 134 |
+
"progress_mae": 0.19023226698239645,
|
| 135 |
+
"rank_acc": 0.8255469501018524,
|
| 136 |
+
"rank_loss": 0.4960531195004781,
|
| 137 |
+
"regret_mae": 0.42156532737943864,
|
| 138 |
+
"success_accuracy": 0.8807473679383596,
|
| 139 |
+
"total_loss": 1.561984618504842
|
| 140 |
+
},
|
| 141 |
+
"per_task": {
|
| 142 |
+
"LiftPegUpright-v1": {
|
| 143 |
+
"action_mse_to_best": 0.32611669615669325,
|
| 144 |
+
"expert_success_rate": 0.8584070796460177,
|
| 145 |
+
"num_groups": 113,
|
| 146 |
+
"oracle_success_rate": 0.9380530973451328,
|
| 147 |
+
"policy_expert_regret": 0.987694250535121,
|
| 148 |
+
"policy_oracle_regret": 1.066896405631462,
|
| 149 |
+
"policy_rollout_progress": 0.6352153201546289,
|
| 150 |
+
"policy_rollout_success_rate": 0.23008849557522124,
|
| 151 |
+
"restore_max_error": 4.76837158203125e-07
|
| 152 |
+
},
|
| 153 |
+
"PickCube-v1": {
|
| 154 |
+
"action_mse_to_best": 0.32144798899231397,
|
| 155 |
+
"expert_success_rate": 0.9402173913043478,
|
| 156 |
+
"num_groups": 184,
|
| 157 |
+
"oracle_success_rate": 0.9456521739130435,
|
| 158 |
+
"policy_expert_regret": 1.0631314818797957,
|
| 159 |
+
"policy_oracle_regret": 1.063155486354726,
|
| 160 |
+
"policy_rollout_progress": 0.5766355847362834,
|
| 161 |
+
"policy_rollout_success_rate": 0.29891304347826086,
|
| 162 |
+
"restore_max_error": 4.76837158203125e-07
|
| 163 |
+
},
|
| 164 |
+
"PullCube-v1": {
|
| 165 |
+
"action_mse_to_best": 0.6065326184034348,
|
| 166 |
+
"expert_success_rate": 0.25,
|
| 167 |
+
"num_groups": 76,
|
| 168 |
+
"oracle_success_rate": 0.40789473684210525,
|
| 169 |
+
"policy_expert_regret": 0.27259286855788606,
|
| 170 |
+
"policy_oracle_regret": 0.5192034780783089,
|
| 171 |
+
"policy_rollout_progress": 0.29394417638449294,
|
| 172 |
+
"policy_rollout_success_rate": 0.18421052631578946,
|
| 173 |
+
"restore_max_error": 4.76837158203125e-07
|
| 174 |
+
},
|
| 175 |
+
"PushCube-v1": {
|
| 176 |
+
"action_mse_to_best": 0.36531390534999136,
|
| 177 |
+
"expert_success_rate": 0.8198198198198198,
|
| 178 |
+
"num_groups": 111,
|
| 179 |
+
"oracle_success_rate": 1.0,
|
| 180 |
+
"policy_expert_regret": 0.3476062245197124,
|
| 181 |
+
"policy_oracle_regret": 0.3810867315477079,
|
| 182 |
+
"policy_rollout_progress": 0.8171114666504903,
|
| 183 |
+
"policy_rollout_success_rate": 0.8018018018018018,
|
| 184 |
+
"restore_max_error": 4.76837158203125e-07
|
| 185 |
+
},
|
| 186 |
+
"StackCube-v1": {
|
| 187 |
+
"action_mse_to_best": 0.5046135181724861,
|
| 188 |
+
"expert_success_rate": 0.6923076923076923,
|
| 189 |
+
"num_groups": 91,
|
| 190 |
+
"oracle_success_rate": 0.8571428571428571,
|
| 191 |
+
"policy_expert_regret": 1.1275780168208447,
|
| 192 |
+
"policy_oracle_regret": 1.2997257709503174,
|
| 193 |
+
"policy_rollout_progress": 0.39087955620917647,
|
| 194 |
+
"policy_rollout_success_rate": 0.15384615384615385,
|
| 195 |
+
"restore_max_error": 3.948807716369629e-07
|
| 196 |
+
}
|
| 197 |
+
}
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"seed": 2,
|
| 201 |
+
"path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_knn8_scale0p35_safe_types_margin0p20.json",
|
| 202 |
+
"num_groups": 575,
|
| 203 |
+
"selection_mode": "retrieval_residual",
|
| 204 |
+
"num_candidates": 128,
|
| 205 |
+
"candidate_sigma": 0.0,
|
| 206 |
+
"selection_margin": 0.2,
|
| 207 |
+
"field_optim_steps": 0,
|
| 208 |
+
"field_optim_step_size": 0.0,
|
| 209 |
+
"field_optim_trust_radius": 0.0,
|
| 210 |
+
"field_optim_l2_penalty": 0.0,
|
| 211 |
+
"retrieval_neighbors": 8,
|
| 212 |
+
"retrieval_metric": "raw",
|
| 213 |
+
"retrieval_type_min_success": 0.0,
|
| 214 |
+
"retrieval_residual_scale": 0.35,
|
| 215 |
+
"retrieval_residual_anchor": "expert",
|
| 216 |
+
"policy_rollout_success_rate": 0.3652173913043478,
|
| 217 |
+
"policy_rollout_progress": 0.5758293948603291,
|
| 218 |
+
"oracle_success_rate": 0.8765217391304347,
|
| 219 |
+
"action_mse_to_best": 0.42547670019466594,
|
| 220 |
+
"best_policy_val": {
|
| 221 |
+
"bc_loss": 0.11367896075050037,
|
| 222 |
+
"field_effect_loss": 0.009670218582161598,
|
| 223 |
+
"field_potential_loss": 0.2641640139950646,
|
| 224 |
+
"field_preference_loss": 0.5130490180518892,
|
| 225 |
+
"lattice_edges": 3833.3333333333335,
|
| 226 |
+
"progress_mae": 0.2021729110015763,
|
| 227 |
+
"rank_acc": 0.8333857821093665,
|
| 228 |
+
"rank_loss": 0.5130119257503085,
|
| 229 |
+
"regret_mae": 0.3958987047274907,
|
| 230 |
+
"success_accuracy": 0.8680730561415354,
|
| 231 |
+
"total_loss": 1.4394984311527677
|
| 232 |
+
},
|
| 233 |
+
"per_task": {
|
| 234 |
+
"LiftPegUpright-v1": {
|
| 235 |
+
"action_mse_to_best": 0.3541048694605706,
|
| 236 |
+
"expert_success_rate": 0.8229166666666666,
|
| 237 |
+
"num_groups": 96,
|
| 238 |
+
"oracle_success_rate": 0.9270833333333334,
|
| 239 |
+
"policy_expert_regret": 0.8651577805479368,
|
| 240 |
+
"policy_oracle_regret": 0.9714324958622456,
|
| 241 |
+
"policy_rollout_progress": 0.639259360730648,
|
| 242 |
+
"policy_rollout_success_rate": 0.3229166666666667,
|
| 243 |
+
"restore_max_error": 3.5762786865234375e-07
|
| 244 |
+
},
|
| 245 |
+
"PickCube-v1": {
|
| 246 |
+
"action_mse_to_best": 0.33806518971393207,
|
| 247 |
+
"expert_success_rate": 0.9444444444444444,
|
| 248 |
+
"num_groups": 198,
|
| 249 |
+
"oracle_success_rate": 0.9595959595959596,
|
| 250 |
+
"policy_expert_regret": 0.9789636366520867,
|
| 251 |
+
"policy_oracle_regret": 0.9860470871807951,
|
| 252 |
+
"policy_rollout_progress": 0.6303682329904552,
|
| 253 |
+
"policy_rollout_success_rate": 0.3383838383838384,
|
| 254 |
+
"restore_max_error": 4.76837158203125e-07
|
| 255 |
+
},
|
| 256 |
+
"PullCube-v1": {
|
| 257 |
+
"action_mse_to_best": 0.674260652396414,
|
| 258 |
+
"expert_success_rate": 0.24444444444444444,
|
| 259 |
+
"num_groups": 90,
|
| 260 |
+
"oracle_success_rate": 0.4666666666666667,
|
| 261 |
+
"policy_expert_regret": 0.25916070034381844,
|
| 262 |
+
"policy_oracle_regret": 0.525496166703043,
|
| 263 |
+
"policy_rollout_progress": 0.3249923576842371,
|
| 264 |
+
"policy_rollout_success_rate": 0.2222222222222222,
|
| 265 |
+
"restore_max_error": 4.0978193283081055e-07
|
| 266 |
+
},
|
| 267 |
+
"PushCube-v1": {
|
| 268 |
+
"action_mse_to_best": 0.38661552500901836,
|
| 269 |
+
"expert_success_rate": 0.8514851485148515,
|
| 270 |
+
"num_groups": 101,
|
| 271 |
+
"oracle_success_rate": 1.0,
|
| 272 |
+
"policy_expert_regret": 0.381283899787629,
|
| 273 |
+
"policy_oracle_regret": 0.4539335219576807,
|
| 274 |
+
"policy_rollout_progress": 0.7836902404185568,
|
| 275 |
+
"policy_rollout_success_rate": 0.7623762376237624,
|
| 276 |
+
"restore_max_error": 4.76837158203125e-07
|
| 277 |
+
},
|
| 278 |
+
"StackCube-v1": {
|
| 279 |
+
"action_mse_to_best": 0.4887388982085718,
|
| 280 |
+
"expert_success_rate": 0.7666666666666667,
|
| 281 |
+
"num_groups": 90,
|
| 282 |
+
"oracle_success_rate": 0.9111111111111111,
|
| 283 |
+
"policy_expert_regret": 1.1605198020736376,
|
| 284 |
+
"policy_oracle_regret": 1.3095919671985838,
|
| 285 |
+
"policy_rollout_progress": 0.4057562978731261,
|
| 286 |
+
"policy_rollout_success_rate": 0.16666666666666666,
|
| 287 |
+
"restore_max_error": 4.76837158203125e-07
|
| 288 |
+
}
|
| 289 |
+
}
|
| 290 |
+
}
|
| 291 |
+
]
|
| 292 |
+
}
|
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn8_scale0p35_safe_types_margin0p20_summary.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# h=16 Best-Policy Checkpoint Rollout
|
| 2 |
+
|
| 3 |
+
Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
|
| 4 |
+
Objective: `near_miss_policy_bc5`
|
| 5 |
+
Result file: `policy_rollout_retrieval_residual_knn8_scale0p35_safe_types_margin0p20.json`
|
| 6 |
+
Completed seeds: 3
|
| 7 |
+
Baseline h=4 policy success: 29.67%
|
| 8 |
+
Baseline h=16 rank-checkpoint success: 29.74%
|
| 9 |
+
|
| 10 |
+
Mean success: 34.84% +/- 1.52%
|
| 11 |
+
Gain vs h=16 rank checkpoint: +5.10%
|
| 12 |
+
Mean progress: 56.53%
|
| 13 |
+
Mean action MSE to best: 0.404
|
| 14 |
+
|
| 15 |
+
| seed | mode | k | retrieval K | retrieval metric | residual anchor | min type success | residual scale | margin | sigma | opt steps | trust | success | progress | oracle | action MSE |
|
| 16 |
+
|---:|---|---:|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
| 17 |
+
| 0 | retrieval_residual | 128 | 8 | raw | expert | 0.00 | 0.35 | 0.200 | 0.00 | 0 | 0.00 | 33.57% | 55.21% | 85.74% | 0.389 |
|
| 18 |
+
| 1 | retrieval_residual | 128 | 8 | raw | expert | 0.00 | 0.35 | 0.200 | 0.00 | 0 | 0.00 | 34.43% | 56.78% | 86.96% | 0.398 |
|
| 19 |
+
| 2 | retrieval_residual | 128 | 8 | raw | expert | 0.00 | 0.35 | 0.200 | 0.00 | 0 | 0.00 | 36.52% | 57.58% | 87.65% | 0.425 |
|
results/paper_table_status.json
CHANGED
|
@@ -498,6 +498,25 @@
|
|
| 498 |
"best_config": null,
|
| 499 |
"gain_vs_h16_policy": 0.051014492753623186
|
| 500 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 501 |
{
|
| 502 |
"key": "retrieval_residual_policy_anchor_scale035_safe",
|
| 503 |
"label": "Policy-relative train-state residual retrieval, scale 0.35, safe non-expert residuals",
|
|
@@ -956,23 +975,23 @@
|
|
| 956 |
}
|
| 957 |
],
|
| 958 |
"best_clean": {
|
| 959 |
-
"key": "
|
| 960 |
-
"label": "
|
| 961 |
-
"path": "
|
| 962 |
"clean_deployment": "yes",
|
| 963 |
"same_state_proposals": "no",
|
| 964 |
"expert_proposal": "no",
|
| 965 |
-
"story_role": "counterfactual advantage abstention",
|
| 966 |
"fallback_success": null,
|
| 967 |
-
"pending_job": "
|
| 968 |
"path_exists": true,
|
| 969 |
"status": "complete",
|
| 970 |
-
"success": 0.
|
| 971 |
-
"std_success": 0.
|
| 972 |
"completed_seeds": null,
|
| 973 |
"num_completed": 3,
|
| 974 |
"best_config": null,
|
| 975 |
-
"gain_vs_h16_policy": 0.
|
| 976 |
},
|
| 977 |
"best_mechanism_no_expert": {
|
| 978 |
"key": "no_expert_lattice",
|
|
@@ -997,7 +1016,7 @@
|
|
| 997 |
"Use no-expert same-state lattice as the conservative mechanism result, not as deployment-clean inference.",
|
| 998 |
"Use full lattice only as an upper result because it includes expert proposals.",
|
| 999 |
"Do not claim external SOTA from this table alone; add current external baselines separately.",
|
| 1000 |
-
"Current best clean deployment row is
|
| 1001 |
"Trust-region field optimization should be framed as a negative/diagnostic ablation.",
|
| 1002 |
"Train-state counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.",
|
| 1003 |
"KNN counterfactual residual retrieval is a positive clean bridge but remains below the current clean best."
|
|
|
|
| 498 |
"best_config": null,
|
| 499 |
"gain_vs_h16_policy": 0.051014492753623186
|
| 500 |
},
|
| 501 |
+
{
|
| 502 |
+
"key": "retrieval_residual_knn2_scale040_safe_margin020",
|
| 503 |
+
"label": "K2 train-state residual retrieval, scale 0.40, safe residuals, advantage margin 0.20",
|
| 504 |
+
"path": "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p40_safe_types_margin0p20_summary.json",
|
| 505 |
+
"clean_deployment": "yes",
|
| 506 |
+
"same_state_proposals": "no",
|
| 507 |
+
"expert_proposal": "no",
|
| 508 |
+
"story_role": "best clean counterfactual advantage abstention",
|
| 509 |
+
"fallback_success": null,
|
| 510 |
+
"pending_job": "14862936/14862937",
|
| 511 |
+
"path_exists": true,
|
| 512 |
+
"status": "complete",
|
| 513 |
+
"success": 0.3501449275362319,
|
| 514 |
+
"std_success": 0.016159257814221867,
|
| 515 |
+
"completed_seeds": null,
|
| 516 |
+
"num_completed": 3,
|
| 517 |
+
"best_config": null,
|
| 518 |
+
"gain_vs_h16_policy": 0.05275362318840582
|
| 519 |
+
},
|
| 520 |
{
|
| 521 |
"key": "retrieval_residual_policy_anchor_scale035_safe",
|
| 522 |
"label": "Policy-relative train-state residual retrieval, scale 0.35, safe non-expert residuals",
|
|
|
|
| 975 |
}
|
| 976 |
],
|
| 977 |
"best_clean": {
|
| 978 |
+
"key": "retrieval_residual_knn2_scale040_safe_margin020",
|
| 979 |
+
"label": "K2 train-state residual retrieval, scale 0.40, safe residuals, advantage margin 0.20",
|
| 980 |
+
"path": "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p40_safe_types_margin0p20_summary.json",
|
| 981 |
"clean_deployment": "yes",
|
| 982 |
"same_state_proposals": "no",
|
| 983 |
"expert_proposal": "no",
|
| 984 |
+
"story_role": "best clean counterfactual advantage abstention",
|
| 985 |
"fallback_success": null,
|
| 986 |
+
"pending_job": "14862936/14862937",
|
| 987 |
"path_exists": true,
|
| 988 |
"status": "complete",
|
| 989 |
+
"success": 0.3501449275362319,
|
| 990 |
+
"std_success": 0.016159257814221867,
|
| 991 |
"completed_seeds": null,
|
| 992 |
"num_completed": 3,
|
| 993 |
"best_config": null,
|
| 994 |
+
"gain_vs_h16_policy": 0.05275362318840582
|
| 995 |
},
|
| 996 |
"best_mechanism_no_expert": {
|
| 997 |
"key": "no_expert_lattice",
|
|
|
|
| 1016 |
"Use no-expert same-state lattice as the conservative mechanism result, not as deployment-clean inference.",
|
| 1017 |
"Use full lattice only as an upper result because it includes expert proposals.",
|
| 1018 |
"Do not claim external SOTA from this table alone; add current external baselines separately.",
|
| 1019 |
+
"Current best clean deployment row is K2 train-state residual retrieval, scale 0.40, safe residuals, advantage margin 0.20 at 35.01%.",
|
| 1020 |
"Trust-region field optimization should be framed as a negative/diagnostic ablation.",
|
| 1021 |
"Train-state counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.",
|
| 1022 |
"KNN counterfactual residual retrieval is a positive clean bridge but remains below the current clean best."
|
results/paper_table_status.md
CHANGED
|
@@ -29,6 +29,7 @@ Baseline h=16 policy: 29.74%
|
|
| 29 |
| retrieval_residual_scale035_safe_types | Train-state residual retrieval, scale 0.35, policy/no-op/wrong-gripper residuals | complete | 33.74% | +4.00 pp | yes | no | no | typed tangent scale fine sweep |
|
| 30 |
| retrieval_residual_scale035_safe_margin020 | Train-state residual retrieval, scale 0.35, safe residuals, advantage margin 0.20 | complete | 34.84% | +5.10 pp | yes | no | no | counterfactual advantage abstention |
|
| 31 |
| retrieval_residual_scale050_safe_margin020 | Train-state residual retrieval, scale 0.50, safe residuals, advantage margin 0.20 | complete | 34.84% | +5.10 pp | yes | no | no | counterfactual advantage abstention scale tie |
|
|
|
|
| 32 |
| retrieval_residual_policy_anchor_scale035_safe | Policy-relative train-state residual retrieval, scale 0.35, safe non-expert residuals | complete | 33.74% | +4.00 pp | yes | no | no | policy-relative tangent anchor diagnostic |
|
| 33 |
| retrieval_residual_scale030_safe_types | Train-state residual retrieval, scale 0.30, policy/no-op/wrong-gripper residuals | complete | 33.51% | +3.77 pp | yes | no | no | typed tangent scale zoom sweep |
|
| 34 |
| retrieval_residual_scale0325_safe_types | Train-state residual retrieval, scale 0.325, policy/no-op/wrong-gripper residuals | complete | 33.74% | +4.00 pp | yes | no | no | typed tangent scale zoom sweep |
|
|
@@ -59,7 +60,7 @@ Baseline h=16 policy: 29.74%
|
|
| 59 |
- Use no-expert same-state lattice as the conservative mechanism result, not as deployment-clean inference.
|
| 60 |
- Use full lattice only as an upper result because it includes expert proposals.
|
| 61 |
- Do not claim external SOTA from this table alone; add current external baselines separately.
|
| 62 |
-
- Current best clean deployment row is
|
| 63 |
- Trust-region field optimization should be framed as a negative/diagnostic ablation.
|
| 64 |
- Train-state counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.
|
| 65 |
- KNN counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.
|
|
|
|
| 29 |
| retrieval_residual_scale035_safe_types | Train-state residual retrieval, scale 0.35, policy/no-op/wrong-gripper residuals | complete | 33.74% | +4.00 pp | yes | no | no | typed tangent scale fine sweep |
|
| 30 |
| retrieval_residual_scale035_safe_margin020 | Train-state residual retrieval, scale 0.35, safe residuals, advantage margin 0.20 | complete | 34.84% | +5.10 pp | yes | no | no | counterfactual advantage abstention |
|
| 31 |
| retrieval_residual_scale050_safe_margin020 | Train-state residual retrieval, scale 0.50, safe residuals, advantage margin 0.20 | complete | 34.84% | +5.10 pp | yes | no | no | counterfactual advantage abstention scale tie |
|
| 32 |
+
| retrieval_residual_knn2_scale040_safe_margin020 | K2 train-state residual retrieval, scale 0.40, safe residuals, advantage margin 0.20 | complete | 35.01% | +5.28 pp | yes | no | no | best clean counterfactual advantage abstention |
|
| 33 |
| retrieval_residual_policy_anchor_scale035_safe | Policy-relative train-state residual retrieval, scale 0.35, safe non-expert residuals | complete | 33.74% | +4.00 pp | yes | no | no | policy-relative tangent anchor diagnostic |
|
| 34 |
| retrieval_residual_scale030_safe_types | Train-state residual retrieval, scale 0.30, policy/no-op/wrong-gripper residuals | complete | 33.51% | +3.77 pp | yes | no | no | typed tangent scale zoom sweep |
|
| 35 |
| retrieval_residual_scale0325_safe_types | Train-state residual retrieval, scale 0.325, policy/no-op/wrong-gripper residuals | complete | 33.74% | +4.00 pp | yes | no | no | typed tangent scale zoom sweep |
|
|
|
|
| 60 |
- Use no-expert same-state lattice as the conservative mechanism result, not as deployment-clean inference.
|
| 61 |
- Use full lattice only as an upper result because it includes expert proposals.
|
| 62 |
- Do not claim external SOTA from this table alone; add current external baselines separately.
|
| 63 |
+
- Current best clean deployment row is K2 train-state residual retrieval, scale 0.40, safe residuals, advantage margin 0.20 at 35.01%.
|
| 64 |
- Trust-region field optimization should be framed as a negative/diagnostic ablation.
|
| 65 |
- Train-state counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.
|
| 66 |
- KNN counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.
|
scripts/build_paper_table_status.py
CHANGED
|
@@ -275,6 +275,16 @@ SPECS = [
|
|
| 275 |
story_role="counterfactual advantage abstention scale tie",
|
| 276 |
pending_job="14862802/14862803",
|
| 277 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 278 |
ResultSpec(
|
| 279 |
key="retrieval_residual_policy_anchor_scale035_safe",
|
| 280 |
label="Policy-relative train-state residual retrieval, scale 0.35, safe non-expert residuals",
|
|
|
|
| 275 |
story_role="counterfactual advantage abstention scale tie",
|
| 276 |
pending_job="14862802/14862803",
|
| 277 |
),
|
| 278 |
+
ResultSpec(
|
| 279 |
+
key="retrieval_residual_knn2_scale040_safe_margin020",
|
| 280 |
+
label="K2 train-state residual retrieval, scale 0.40, safe residuals, advantage margin 0.20",
|
| 281 |
+
path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p40_safe_types_margin0p20_summary.json",
|
| 282 |
+
clean_deployment="yes",
|
| 283 |
+
same_state_proposals="no",
|
| 284 |
+
expert_proposal="no",
|
| 285 |
+
story_role="best clean counterfactual advantage abstention",
|
| 286 |
+
pending_job="14862936/14862937",
|
| 287 |
+
),
|
| 288 |
ResultSpec(
|
| 289 |
key="retrieval_residual_policy_anchor_scale035_safe",
|
| 290 |
label="Policy-relative train-state residual retrieval, scale 0.35, safe non-expert residuals",
|