Auto-sync: 2026-06-28 02:44:48 (part 2)

results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p30_safe_types_summary.json

@@ -0,0 +1,286 @@
+{
+  "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
+  "objective": "near_miss_policy_bc5",
+  "out_name": "policy_rollout_retrieval_residual_scale0p30_safe_types.json",
+  "num_completed": 3,
+  "baseline_h4_policy_success": 0.2967,
+  "baseline_h16_rank_checkpoint_success": 0.29739130434782607,
+  "mean_success": 0.33507246376811595,
+  "std_success": 0.005020437123388053,
+  "mean_progress": 0.5540134637511295,
+  "mean_action_mse_to_best": 0.4067905238411133,
+  "gain_vs_h4": 0.03837246376811593,
+  "gain_vs_h16_rank_checkpoint": 0.03768115942028988,
+  "rows": [
+    {
+      "seed": 0,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_scale0p30_safe_types.json",
+      "num_groups": 575,
+      "selection_mode": "retrieval_residual",
+      "num_candidates": 16,
+      "candidate_sigma": 0.0,
+      "field_optim_steps": 0,
+      "field_optim_step_size": 0.0,
+      "field_optim_trust_radius": 0.0,
+      "field_optim_l2_penalty": 0.0,
+      "retrieval_neighbors": 1,
+      "retrieval_metric": "raw",
+      "retrieval_type_min_success": 0.0,
+      "retrieval_residual_scale": 0.3,
+      "policy_rollout_success_rate": 0.33217391304347826,
+      "policy_rollout_progress": 0.5444753354949796,
+      "oracle_success_rate": 0.8573913043478261,
+      "action_mse_to_best": 0.3888626198089965,
+      "best_policy_val": {
+        "bc_loss": 0.13721593966086706,
+        "field_effect_loss": 0.009290305380192068,
+        "field_potential_loss": 0.2666468388504452,
+        "field_preference_loss": 0.5130573478009965,
+        "lattice_edges": 3833.3333333333335,
+        "progress_mae": 0.1933159919248687,
+        "rank_acc": 0.8265031774838766,
+        "rank_loss": 0.5130523675017886,
+        "regret_mae": 0.3756548762321472,
+        "success_accuracy": 0.8773836526605818,
+        "total_loss": 1.5581054819954767
+      },
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.3208054037351814,
+          "expert_success_rate": 0.8865979381443299,
+          "num_groups": 97,
+          "oracle_success_rate": 0.9278350515463918,
+          "policy_expert_regret": 1.0926802073864592,
+          "policy_oracle_regret": 1.119095685094902,
+          "policy_rollout_progress": 0.567425964572995,
+          "policy_rollout_success_rate": 0.21649484536082475,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.307855731771829,
+          "expert_success_rate": 0.9375,
+          "num_groups": 208,
+          "oracle_success_rate": 0.9471153846153846,
+          "policy_expert_regret": 1.0262822506035894,
+          "policy_oracle_regret": 1.0397051180300947,
+          "policy_rollout_progress": 0.584638239377035,
+          "policy_rollout_success_rate": 0.3076923076923077,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.6046535208240732,
+          "expert_success_rate": 0.19480519480519481,
+          "num_groups": 77,
+          "oracle_success_rate": 0.36363636363636365,
+          "policy_expert_regret": 0.19415880497786905,
+          "policy_oracle_regret": 0.3655842968879582,
+          "policy_rollout_progress": 0.34888698388036193,
+          "policy_rollout_success_rate": 0.24675324675324675,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.3696133341798757,
+          "expert_success_rate": 0.8617021276595744,
+          "num_groups": 94,
+          "oracle_success_rate": 0.9893617021276596,
+          "policy_expert_regret": 0.5720039699622925,
+          "policy_oracle_regret": 0.6495547312054228,
+          "policy_rollout_progress": 0.6908708007094708,
+          "policy_rollout_success_rate": 0.6595744680851063,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.4761809646763434,
+          "expert_success_rate": 0.6767676767676768,
+          "num_groups": 99,
+          "oracle_success_rate": 0.8585858585858586,
+          "policy_expert_regret": 0.954742374895799,
+          "policy_oracle_regret": 1.1176562870692726,
+          "policy_rollout_progress": 0.450728207375064,
+          "policy_rollout_success_rate": 0.25252525252525254,
+          "restore_max_error": 3.948807716369629e-07
+        }
+      }
+    },
+    {
+      "seed": 1,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_scale0p30_safe_types.json",
+      "num_groups": 575,
+      "selection_mode": "retrieval_residual",
+      "num_candidates": 16,
+      "candidate_sigma": 0.0,
+      "field_optim_steps": 0,
+      "field_optim_step_size": 0.0,
+      "field_optim_trust_radius": 0.0,
+      "field_optim_l2_penalty": 0.0,
+      "retrieval_neighbors": 1,
+      "retrieval_metric": "raw",
+      "retrieval_type_min_success": 0.0,
+      "retrieval_residual_scale": 0.3,
+      "policy_rollout_success_rate": 0.33217391304347826,
+      "policy_rollout_progress": 0.5561610209990455,
+      "oracle_success_rate": 0.8695652173913043,
+      "action_mse_to_best": 0.3987280774731999,
+      "best_policy_val": {
+        "bc_loss": 0.1315989200439718,
+        "field_effect_loss": 0.010110370814800262,
+        "field_potential_loss": 0.31546536915832096,
+        "field_preference_loss": 0.4960637440284093,
+        "lattice_edges": 3833.3333333333335,
+        "progress_mae": 0.19023226698239645,
+        "rank_acc": 0.8255469501018524,
+        "rank_loss": 0.4960531195004781,
+        "regret_mae": 0.42156532737943864,
+        "success_accuracy": 0.8807473679383596,
+        "total_loss": 1.561984618504842
+      },
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.33368760702587597,
+          "expert_success_rate": 0.8584070796460177,
+          "num_groups": 113,
+          "oracle_success_rate": 0.9380530973451328,
+          "policy_expert_regret": 1.0038910630793698,
+          "policy_oracle_regret": 1.0833192215820329,
+          "policy_rollout_progress": 0.6252008696309234,
+          "policy_rollout_success_rate": 0.21238938053097345,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.33354028644364164,
+          "expert_success_rate": 0.9402173913043478,
+          "num_groups": 184,
+          "oracle_success_rate": 0.9456521739130435,
+          "policy_expert_regret": 1.0729382047986449,
+          "policy_oracle_regret": 1.072962209273575,
+          "policy_rollout_progress": 0.5668288618174341,
+          "policy_rollout_success_rate": 0.29891304347826086,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.5759741171802345,
+          "expert_success_rate": 0.25,
+          "num_groups": 76,
+          "oracle_success_rate": 0.40789473684210525,
+          "policy_expert_regret": 0.3412382446082407,
+          "policy_oracle_regret": 0.5909676778522369,
+          "policy_rollout_progress": 0.2630890922277774,
+          "policy_rollout_success_rate": 0.14473684210526316,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.36793937265604465,
+          "expert_success_rate": 0.8198198198198198,
+          "num_groups": 111,
+          "oracle_success_rate": 1.0,
+          "policy_expert_regret": 0.3472233432101774,
+          "policy_oracle_regret": 0.4141584269635312,
+          "policy_rollout_progress": 0.802057789252685,
+          "policy_rollout_success_rate": 0.7837837837837838,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.500826692055133,
+          "expert_success_rate": 0.6923076923076923,
+          "num_groups": 91,
+          "oracle_success_rate": 0.8571428571428571,
+          "policy_expert_regret": 1.1247741376960672,
+          "policy_oracle_regret": 1.2969218918255396,
+          "policy_rollout_progress": 0.3936834353339541,
+          "policy_rollout_success_rate": 0.15384615384615385,
+          "restore_max_error": 3.948807716369629e-07
+        }
+      }
+    },
+    {
+      "seed": 2,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_scale0p30_safe_types.json",
+      "num_groups": 575,
+      "selection_mode": "retrieval_residual",
+      "num_candidates": 16,
+      "candidate_sigma": 0.0,
+      "field_optim_steps": 0,
+      "field_optim_step_size": 0.0,
+      "field_optim_trust_radius": 0.0,
+      "field_optim_l2_penalty": 0.0,
+      "retrieval_neighbors": 1,
+      "retrieval_metric": "raw",
+      "retrieval_type_min_success": 0.0,
+      "retrieval_residual_scale": 0.3,
+      "policy_rollout_success_rate": 0.3408695652173913,
+      "policy_rollout_progress": 0.5614040347593634,
+      "oracle_success_rate": 0.8765217391304347,
+      "action_mse_to_best": 0.4327808742411435,
+      "best_policy_val": {
+        "bc_loss": 0.11367896075050037,
+        "field_effect_loss": 0.009670218582161598,
+        "field_potential_loss": 0.2641640139950646,
+        "field_preference_loss": 0.5130490180518892,
+        "lattice_edges": 3833.3333333333335,
+        "progress_mae": 0.2021729110015763,
+        "rank_acc": 0.8333857821093665,
+        "rank_loss": 0.5130119257503085,
+        "regret_mae": 0.3958987047274907,
+        "success_accuracy": 0.8680730561415354,
+        "total_loss": 1.4394984311527677
+      },
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.3564206978335278,
+          "expert_success_rate": 0.8229166666666666,
+          "num_groups": 96,
+          "oracle_success_rate": 0.9270833333333334,
+          "policy_expert_regret": 0.8506953514491519,
+          "policy_oracle_regret": 0.9569462323561311,
+          "policy_rollout_progress": 0.6433289575700959,
+          "policy_rollout_success_rate": 0.3333333333333333,
+          "restore_max_error": 3.5762786865234375e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.35353349926270017,
+          "expert_success_rate": 0.9444444444444444,
+          "num_groups": 198,
+          "oracle_success_rate": 0.9595959595959596,
+          "policy_expert_regret": 1.0548599780433707,
+          "policy_oracle_regret": 1.0619434285720792,
+          "policy_rollout_progress": 0.5999264370537165,
+          "policy_rollout_success_rate": 0.29292929292929293,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.6617127334078153,
+          "expert_success_rate": 0.24444444444444444,
+          "num_groups": 90,
+          "oracle_success_rate": 0.4666666666666667,
+          "policy_expert_regret": 0.31624702468721405,
+          "policy_oracle_regret": 0.5234127792736722,
+          "policy_rollout_progress": 0.32531494989784226,
+          "policy_rollout_success_rate": 0.2222222222222222,
+          "restore_max_error": 4.0978193283081055e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.3918927944827788,
+          "expert_success_rate": 0.8514851485148515,
+          "num_groups": 101,
+          "oracle_success_rate": 1.0,
+          "policy_expert_regret": 0.4374538593658126,
+          "policy_oracle_regret": 0.5101034815358644,
+          "policy_rollout_progress": 0.757223251137403,
+          "policy_rollout_success_rate": 0.7326732673267327,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.5055296065906684,
+          "expert_success_rate": 0.7666666666666667,
+          "num_groups": 90,
+          "oracle_success_rate": 0.9111111111111111,
+          "policy_expert_regret": 1.1982895359396935,
+          "policy_oracle_regret": 1.3472699383894602,
+          "policy_rollout_progress": 0.4056045740842819,
+          "policy_rollout_success_rate": 0.13333333333333333,
+          "restore_max_error": 4.76837158203125e-07
+        }
+      }
+    }
+  ]
+}

results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p30_safe_types_summary.md

@@ -0,0 +1,19 @@
+# h=16 Best-Policy Checkpoint Rollout
+
+Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
+Objective: `near_miss_policy_bc5`
+Result file: `policy_rollout_retrieval_residual_scale0p30_safe_types.json`
+Completed seeds: 3
+Baseline h=4 policy success: 29.67%
+Baseline h=16 rank-checkpoint success: 29.74%
+
+Mean success: 33.51% +/- 0.50%
+Gain vs h=16 rank checkpoint: +3.77%
+Mean progress: 55.40%
+Mean action MSE to best: 0.407
+
+| seed | mode | k | retrieval K | retrieval metric | min type success | residual scale | sigma | opt steps | trust | success | progress | oracle | action MSE |
+|---:|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
+| 0 | retrieval_residual | 16 | 1 | raw | 0.00 | 0.30 | 0.00 | 0 | 0.00 | 33.22% | 54.45% | 85.74% | 0.389 |
+| 1 | retrieval_residual | 16 | 1 | raw | 0.00 | 0.30 | 0.00 | 0 | 0.00 | 33.22% | 55.62% | 86.96% | 0.399 |
+| 2 | retrieval_residual | 16 | 1 | raw | 0.00 | 0.30 | 0.00 | 0 | 0.00 | 34.09% | 56.14% | 87.65% | 0.433 |

results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p325_safe_types_summary.json

@@ -0,0 +1,286 @@
+{
+  "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
+  "objective": "near_miss_policy_bc5",
+  "out_name": "policy_rollout_retrieval_residual_scale0p325_safe_types.json",
+  "num_completed": 3,
+  "baseline_h4_policy_success": 0.2967,
+  "baseline_h16_rank_checkpoint_success": 0.29739130434782607,
+  "mean_success": 0.3373913043478261,
+  "std_success": 0.006956521739130445,
+  "mean_progress": 0.5542141509488937,
+  "mean_action_mse_to_best": 0.4088565411100137,
+  "gain_vs_h4": 0.040691304347826085,
+  "gain_vs_h16_rank_checkpoint": 0.040000000000000036,
+  "rows": [
+    {
+      "seed": 0,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_scale0p325_safe_types.json",
+      "num_groups": 575,
+      "selection_mode": "retrieval_residual",
+      "num_candidates": 16,
+      "candidate_sigma": 0.0,
+      "field_optim_steps": 0,
+      "field_optim_step_size": 0.0,
+      "field_optim_trust_radius": 0.0,
+      "field_optim_l2_penalty": 0.0,
+      "retrieval_neighbors": 1,
+      "retrieval_metric": "raw",
+      "retrieval_type_min_success": 0.0,
+      "retrieval_residual_scale": 0.325,
+      "policy_rollout_success_rate": 0.33043478260869563,
+      "policy_rollout_progress": 0.5445710693785678,
+      "oracle_success_rate": 0.8573913043478261,
+      "action_mse_to_best": 0.39044705578895367,
+      "best_policy_val": {
+        "bc_loss": 0.13721593966086706,
+        "field_effect_loss": 0.009290305380192068,
+        "field_potential_loss": 0.2666468388504452,
+        "field_preference_loss": 0.5130573478009965,
+        "lattice_edges": 3833.3333333333335,
+        "progress_mae": 0.1933159919248687,
+        "rank_acc": 0.8265031774838766,
+        "rank_loss": 0.5130523675017886,
+        "regret_mae": 0.3756548762321472,
+        "success_accuracy": 0.8773836526605818,
+        "total_loss": 1.5581054819954767
+      },
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.3209787567779806,
+          "expert_success_rate": 0.8865979381443299,
+          "num_groups": 97,
+          "oracle_success_rate": 0.9278350515463918,
+          "policy_expert_regret": 1.0792337135248578,
+          "policy_oracle_regret": 1.1056491912333006,
+          "policy_rollout_progress": 0.570563180084081,
+          "policy_rollout_success_rate": 0.2268041237113402,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.3094577855752924,
+          "expert_success_rate": 0.9375,
+          "num_groups": 208,
+          "oracle_success_rate": 0.9471153846153846,
+          "policy_expert_regret": 1.02838264804226,
+          "policy_oracle_regret": 1.0351601761425486,
+          "policy_rollout_progress": 0.589183181264581,
+          "policy_rollout_success_rate": 0.3076923076923077,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.6063742484752234,
+          "expert_success_rate": 0.19480519480519481,
+          "num_groups": 77,
+          "oracle_success_rate": 0.36363636363636365,
+          "policy_expert_regret": 0.1962964894225845,
+          "policy_oracle_regret": 0.3678794406934992,
+          "policy_rollout_progress": 0.34610183783746384,
+          "policy_rollout_success_rate": 0.24675324675324675,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.37110239136567774,
+          "expert_success_rate": 0.8617021276595744,
+          "num_groups": 94,
+          "oracle_success_rate": 0.9893617021276596,
+          "policy_expert_regret": 0.5718955254935204,
+          "policy_oracle_regret": 0.6494462867366507,
+          "policy_rollout_progress": 0.6909792451782429,
+          "policy_rollout_success_rate": 0.6595744680851063,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.47909551869250006,
+          "expert_success_rate": 0.6767676767676768,
+          "num_groups": 99,
+          "oracle_success_rate": 0.8585858585858586,
+          "policy_expert_regret": 0.9733010279108779,
+          "policy_oracle_regret": 1.1478618261489002,
+          "policy_rollout_progress": 0.44072468849745666,
+          "policy_rollout_success_rate": 0.23232323232323232,
+          "restore_max_error": 3.948807716369629e-07
+        }
+      }
+    },
+    {
+      "seed": 1,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_scale0p325_safe_types.json",
+      "num_groups": 575,
+      "selection_mode": "retrieval_residual",
+      "num_candidates": 16,
+      "candidate_sigma": 0.0,
+      "field_optim_steps": 0,
+      "field_optim_step_size": 0.0,
+      "field_optim_trust_radius": 0.0,
+      "field_optim_l2_penalty": 0.0,
+      "retrieval_neighbors": 1,
+      "retrieval_metric": "raw",
+      "retrieval_type_min_success": 0.0,
+      "retrieval_residual_scale": 0.325,
+      "policy_rollout_success_rate": 0.3373913043478261,
+      "policy_rollout_progress": 0.5570713255702994,
+      "oracle_success_rate": 0.8695652173913043,
+      "action_mse_to_best": 0.4005185535890253,
+      "best_policy_val": {
+        "bc_loss": 0.1315989200439718,
+        "field_effect_loss": 0.010110370814800262,
+        "field_potential_loss": 0.31546536915832096,
+        "field_preference_loss": 0.4960637440284093,
+        "lattice_edges": 3833.3333333333335,
+        "progress_mae": 0.19023226698239645,
+        "rank_acc": 0.8255469501018524,
+        "rank_loss": 0.4960531195004781,
+        "regret_mae": 0.42156532737943864,
+        "success_accuracy": 0.8807473679383596,
+        "total_loss": 1.561984618504842
+      },
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.3340790134450767,
+          "expert_success_rate": 0.8584070796460177,
+          "num_groups": 113,
+          "oracle_success_rate": 0.9380530973451328,
+          "policy_expert_regret": 0.9935315317284744,
+          "policy_oracle_regret": 1.0842157546925333,
+          "policy_rollout_progress": 0.6243043365204228,
+          "policy_rollout_success_rate": 0.21238938053097345,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.3387653343177036,
+          "expert_success_rate": 0.9402173913043478,
+          "num_groups": 184,
+          "oracle_success_rate": 0.9456521739130435,
+          "policy_expert_regret": 1.0676670129583015,
+          "policy_oracle_regret": 1.0676910174332317,
+          "policy_rollout_progress": 0.568859128195428,
+          "policy_rollout_success_rate": 0.30978260869565216,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.5752073404820341,
+          "expert_success_rate": 0.25,
+          "num_groups": 76,
+          "oracle_success_rate": 0.40789473684210525,
+          "policy_expert_regret": 0.29777104804375804,
+          "policy_oracle_regret": 0.550003365464363,
+          "policy_rollout_progress": 0.2676392478007122,
+          "policy_rollout_success_rate": 0.15789473684210525,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.37022824276674976,
+          "expert_success_rate": 0.8198198198198198,
+          "num_groups": 111,
+          "oracle_success_rate": 1.0,
+          "policy_expert_regret": 0.34810926020145416,
+          "policy_oracle_regret": 0.4149654534485963,
+          "policy_rollout_progress": 0.8012507627676199,
+          "policy_rollout_success_rate": 0.7837837837837838,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.4989376423757646,
+          "expert_success_rate": 0.6923076923076923,
+          "num_groups": 91,
+          "oracle_success_rate": 0.8571428571428571,
+          "policy_expert_regret": 1.1248298248091897,
+          "policy_oracle_regret": 1.2969775789386624,
+          "policy_rollout_progress": 0.39362774822083146,
+          "policy_rollout_success_rate": 0.15384615384615385,
+          "restore_max_error": 3.948807716369629e-07
+        }
+      }
+    },
+    {
+      "seed": 2,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_scale0p325_safe_types.json",
+      "num_groups": 575,
+      "selection_mode": "retrieval_residual",
+      "num_candidates": 16,
+      "candidate_sigma": 0.0,
+      "field_optim_steps": 0,
+      "field_optim_step_size": 0.0,
+      "field_optim_trust_radius": 0.0,
+      "field_optim_l2_penalty": 0.0,
+      "retrieval_neighbors": 1,
+      "retrieval_metric": "raw",
+      "retrieval_type_min_success": 0.0,
+      "retrieval_residual_scale": 0.325,
+      "policy_rollout_success_rate": 0.3443478260869565,
+      "policy_rollout_progress": 0.5610000578978139,
+      "oracle_success_rate": 0.8765217391304347,
+      "action_mse_to_best": 0.43560401395206216,
+      "best_policy_val": {
+        "bc_loss": 0.11367896075050037,
+        "field_effect_loss": 0.009670218582161598,
+        "field_potential_loss": 0.2641640139950646,
+        "field_preference_loss": 0.5130490180518892,
+        "lattice_edges": 3833.3333333333335,
+        "progress_mae": 0.2021729110015763,
+        "rank_acc": 0.8333857821093665,
+        "rank_loss": 0.5130119257503085,
+        "regret_mae": 0.3958987047274907,
+        "success_accuracy": 0.8680730561415354,
+        "total_loss": 1.4394984311527677
+      },
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.3567177759008094,
+          "expert_success_rate": 0.8229166666666666,
+          "num_groups": 96,
+          "oracle_success_rate": 0.9270833333333334,
+          "policy_expert_regret": 0.8647989913200339,
+          "policy_oracle_regret": 0.9710488663986325,
+          "policy_rollout_progress": 0.6396429901942611,
+          "policy_rollout_success_rate": 0.3229166666666667,
+          "restore_max_error": 3.5762786865234375e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.3588028214683737,
+          "expert_success_rate": 0.9444444444444444,
+          "num_groups": 198,
+          "oracle_success_rate": 0.9595959595959596,
+          "policy_expert_regret": 1.0402113473738985,
+          "policy_oracle_regret": 1.047294797902607,
+          "policy_rollout_progress": 0.6044740576221787,
+          "policy_rollout_success_rate": 0.30303030303030304,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.6635678420464198,
+          "expert_success_rate": 0.24444444444444444,
+          "num_groups": 90,
+          "oracle_success_rate": 0.4666666666666667,
+          "policy_expert_regret": 0.29917826053376,
+          "policy_oracle_regret": 0.5254249903580381,
+          "policy_rollout_progress": 0.32371116887985,
+          "policy_rollout_success_rate": 0.2222222222222222,
+          "restore_max_error": 4.0978193283081055e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.3921640517540497,
+          "expert_success_rate": 0.8514851485148515,
+          "num_groups": 101,
+          "oracle_success_rate": 1.0,
+          "policy_expert_regret": 0.43774790958602827,
+          "policy_oracle_regret": 0.51039753175608,
+          "policy_rollout_progress": 0.7569292009171873,
+          "policy_rollout_success_rate": 0.7326732673267327,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.509497420820925,
+          "expert_success_rate": 0.7666666666666667,
+          "num_groups": 90,
+          "oracle_success_rate": 0.9111111111111111,
+          "policy_expert_regret": 1.2065284666087892,
+          "policy_oracle_regret": 1.3428645324375894,
+          "policy_rollout_progress": 0.39888431479533515,
+          "policy_rollout_success_rate": 0.14444444444444443,
+          "restore_max_error": 4.76837158203125e-07
+        }
+      }
+    }
+  ]
+}

results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p325_safe_types_summary.md

@@ -0,0 +1,19 @@
+# h=16 Best-Policy Checkpoint Rollout
+
+Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
+Objective: `near_miss_policy_bc5`
+Result file: `policy_rollout_retrieval_residual_scale0p325_safe_types.json`
+Completed seeds: 3
+Baseline h=4 policy success: 29.67%
+Baseline h=16 rank-checkpoint success: 29.74%
+
+Mean success: 33.74% +/- 0.70%
+Gain vs h=16 rank checkpoint: +4.00%
+Mean progress: 55.42%
+Mean action MSE to best: 0.409
+
+| seed | mode | k | retrieval K | retrieval metric | min type success | residual scale | sigma | opt steps | trust | success | progress | oracle | action MSE |
+|---:|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
+| 0 | retrieval_residual | 16 | 1 | raw | 0.00 | 0.33 | 0.00 | 0 | 0.00 | 33.04% | 54.46% | 85.74% | 0.390 |
+| 1 | retrieval_residual | 16 | 1 | raw | 0.00 | 0.33 | 0.00 | 0 | 0.00 | 33.74% | 55.71% | 86.96% | 0.401 |
+| 2 | retrieval_residual | 16 | 1 | raw | 0.00 | 0.33 | 0.00 | 0 | 0.00 | 34.43% | 56.10% | 87.65% | 0.436 |

results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p375_safe_types_summary.json

@@ -0,0 +1,286 @@
+{
+  "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
+  "objective": "near_miss_policy_bc5",
+  "out_name": "policy_rollout_retrieval_residual_scale0p375_safe_types.json",
+  "num_completed": 3,
+  "baseline_h4_policy_success": 0.2967,
+  "baseline_h16_rank_checkpoint_success": 0.29739130434782607,
+  "mean_success": 0.33507246376811595,
+  "std_success": 0.008753431229299422,
+  "mean_progress": 0.5540295836509651,
+  "mean_action_mse_to_best": 0.4135543125857046,
+  "gain_vs_h4": 0.03837246376811593,
+  "gain_vs_h16_rank_checkpoint": 0.03768115942028988,
+  "rows": [
+    {
+      "seed": 0,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_scale0p375_safe_types.json",
+      "num_groups": 575,
+      "selection_mode": "retrieval_residual",
+      "num_candidates": 16,
+      "candidate_sigma": 0.0,
+      "field_optim_steps": 0,
+      "field_optim_step_size": 0.0,
+      "field_optim_trust_radius": 0.0,
+      "field_optim_l2_penalty": 0.0,
+      "retrieval_neighbors": 1,
+      "retrieval_metric": "raw",
+      "retrieval_type_min_success": 0.0,
+      "retrieval_residual_scale": 0.375,
+      "policy_rollout_success_rate": 0.3269565217391304,
+      "policy_rollout_progress": 0.5432891027694163,
+      "oracle_success_rate": 0.8573913043478261,
+      "action_mse_to_best": 0.39131664984981,
+      "best_policy_val": {
+        "bc_loss": 0.13721593966086706,
+        "field_effect_loss": 0.009290305380192068,
+        "field_potential_loss": 0.2666468388504452,
+        "field_preference_loss": 0.5130573478009965,
+        "lattice_edges": 3833.3333333333335,
+        "progress_mae": 0.1933159919248687,
+        "rank_acc": 0.8265031774838766,
+        "rank_loss": 0.5130523675017886,
+        "regret_mae": 0.3756548762321472,
+        "success_accuracy": 0.8773836526605818,
+        "total_loss": 1.5581054819954767
+      },
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.32084958706070316,
+          "expert_success_rate": 0.8865979381443299,
+          "num_groups": 97,
+          "oracle_success_rate": 0.9278350515463918,
+          "policy_expert_regret": 1.0792337135248578,
+          "policy_oracle_regret": 1.1056491912333006,
+          "policy_rollout_progress": 0.570563180084081,
+          "policy_rollout_success_rate": 0.2268041237113402,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.31058768868947834,
+          "expert_success_rate": 0.9375,
+          "num_groups": 208,
+          "oracle_success_rate": 0.9471153846153846,
+          "policy_expert_regret": 1.031485921004787,
+          "policy_oracle_regret": 1.0449087884312926,
+          "policy_rollout_progress": 0.5842422612835295,
+          "policy_rollout_success_rate": 0.30288461538461536,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.6137808516815111,
+          "expert_success_rate": 0.19480519480519481,
+          "num_groups": 77,
+          "oracle_success_rate": 0.36363636363636365,
+          "policy_expert_regret": 0.21619461752571068,
+          "policy_oracle_regret": 0.38668908493278864,
+          "policy_rollout_progress": 0.33998702461649843,
+          "policy_rollout_success_rate": 0.23376623376623376,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.36967177352214114,
+          "expert_success_rate": 0.8617021276595744,
+          "num_groups": 94,
+          "oracle_success_rate": 0.9893617021276596,
+          "policy_expert_regret": 0.5718626766762835,
+          "policy_oracle_regret": 0.6494134379194137,
+          "policy_rollout_progress": 0.6910120939954798,
+          "policy_rollout_success_rate": 0.6595744680851063,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.47749648667458033,
+          "expert_success_rate": 0.6767676767676768,
+          "num_groups": 99,
+          "oracle_success_rate": 0.8585858585858586,
+          "policy_expert_regret": 0.965505714067305,
+          "policy_oracle_regret": 1.140201892967176,
+          "policy_rollout_progress": 0.4483846216791808,
+          "policy_rollout_success_rate": 0.23232323232323232,
+          "restore_max_error": 3.948807716369629e-07
+        }
+      }
+    },
+    {
+      "seed": 1,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_scale0p375_safe_types.json",
+      "num_groups": 575,
+      "selection_mode": "retrieval_residual",
+      "num_candidates": 16,
+      "candidate_sigma": 0.0,
+      "field_optim_steps": 0,
+      "field_optim_step_size": 0.0,
+      "field_optim_trust_radius": 0.0,
+      "field_optim_l2_penalty": 0.0,
+      "retrieval_neighbors": 1,
+      "retrieval_metric": "raw",
+      "retrieval_type_min_success": 0.0,
+      "retrieval_residual_scale": 0.375,
+      "policy_rollout_success_rate": 0.3339130434782609,
+      "policy_rollout_progress": 0.5560864147177452,
+      "oracle_success_rate": 0.8695652173913043,
+      "action_mse_to_best": 0.4061409263409998,
+      "best_policy_val": {
+        "bc_loss": 0.1315989200439718,
+        "field_effect_loss": 0.010110370814800262,
+        "field_potential_loss": 0.31546536915832096,
+        "field_preference_loss": 0.4960637440284093,
+        "lattice_edges": 3833.3333333333335,
+        "progress_mae": 0.19023226698239645,
+        "rank_acc": 0.8255469501018524,
+        "rank_loss": 0.4960531195004781,
+        "regret_mae": 0.42156532737943864,
+        "success_accuracy": 0.8807473679383596,
+        "total_loss": 1.561984618504842
+      },
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.3342079682671259,
+          "expert_success_rate": 0.8584070796460177,
+          "num_groups": 113,
+          "oracle_success_rate": 0.9380530973451328,
+          "policy_expert_regret": 1.0068653348536618,
+          "policy_oracle_regret": 1.0866819009580444,
+          "policy_rollout_progress": 0.6218381902549119,
+          "policy_rollout_success_rate": 0.21238938053097345,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.34539582554008,
+          "expert_success_rate": 0.9402173913043478,
+          "num_groups": 184,
+          "oracle_success_rate": 0.9456521739130435,
+          "policy_expert_regret": 1.0724786393263417,
+          "policy_oracle_regret": 1.072502643801272,
+          "policy_rollout_progress": 0.5694822844360834,
+          "policy_rollout_success_rate": 0.30434782608695654,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.5809329142303843,
+          "expert_success_rate": 0.25,
+          "num_groups": 76,
+          "oracle_success_rate": 0.40789473684210525,
+          "policy_expert_regret": 0.2983962231588329,
+          "policy_oracle_regret": 0.5503765984612983,
+          "policy_rollout_progress": 0.2670566699587078,
+          "policy_rollout_success_rate": 0.15789473684210525,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.37470649330465644,
+          "expert_success_rate": 0.8198198198198198,
+          "num_groups": 111,
+          "oracle_success_rate": 1.0,
+          "policy_expert_regret": 0.3638223847290417,
+          "policy_oracle_regret": 0.43040637134968696,
+          "policy_rollout_progress": 0.7948188538755383,
+          "policy_rollout_success_rate": 0.7747747747747747,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.510652506529548,
+          "expert_success_rate": 0.6923076923076923,
+          "num_groups": 91,
+          "oracle_success_rate": 0.8571428571428571,
+          "policy_expert_regret": 1.1209187496166964,
+          "policy_oracle_regret": 1.293066503746169,
+          "policy_rollout_progress": 0.39753882341332486,
+          "policy_rollout_success_rate": 0.15384615384615385,
+          "restore_max_error": 3.948807716369629e-07
+        }
+      }
+    },
+    {
+      "seed": 2,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_scale0p375_safe_types.json",
+      "num_groups": 575,
+      "selection_mode": "retrieval_residual",
+      "num_candidates": 16,
+      "candidate_sigma": 0.0,
+      "field_optim_steps": 0,
+      "field_optim_step_size": 0.0,
+      "field_optim_trust_radius": 0.0,
+      "field_optim_l2_penalty": 0.0,
+      "retrieval_neighbors": 1,
+      "retrieval_metric": "raw",
+      "retrieval_type_min_success": 0.0,
+      "retrieval_residual_scale": 0.375,
+      "policy_rollout_success_rate": 0.3443478260869565,
+      "policy_rollout_progress": 0.5627132334657338,
+      "oracle_success_rate": 0.8765217391304347,
+      "action_mse_to_best": 0.44320536156630386,
+      "best_policy_val": {
+        "bc_loss": 0.11367896075050037,
+        "field_effect_loss": 0.009670218582161598,
+        "field_potential_loss": 0.2641640139950646,
+        "field_preference_loss": 0.5130490180518892,
+        "lattice_edges": 3833.3333333333335,
+        "progress_mae": 0.2021729110015763,
+        "rank_acc": 0.8333857821093665,
+        "rank_loss": 0.5130119257503085,
+        "regret_mae": 0.3958987047274907,
+        "success_accuracy": 0.8680730561415354,
+        "total_loss": 1.4394984311527677
+      },
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.35788121770504705,
+          "expert_success_rate": 0.8229166666666666,
+          "num_groups": 96,
+          "oracle_success_rate": 0.9270833333333334,
+          "policy_expert_regret": 0.8651992557570338,
+          "policy_oracle_regret": 0.9582117134705186,
+          "policy_rollout_progress": 0.6420634764557084,
+          "policy_rollout_success_rate": 0.3333333333333333,
+          "restore_max_error": 3.5762786865234375e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.36824658943923433,
+          "expert_success_rate": 0.9444444444444444,
+          "num_groups": 198,
+          "oracle_success_rate": 0.9595959595959596,
+          "policy_expert_regret": 1.040687483086279,
+          "policy_oracle_regret": 1.0477709336149874,
+          "policy_rollout_progress": 0.6039979219097983,
+          "policy_rollout_success_rate": 0.30303030303030304,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.6755460912982623,
+          "expert_success_rate": 0.24444444444444444,
+          "num_groups": 90,
+          "oracle_success_rate": 0.4666666666666667,
+          "policy_expert_regret": 0.301549016787774,
+          "policy_oracle_regret": 0.5462499696761369,
+          "policy_rollout_progress": 0.3145351243101888,
+          "policy_rollout_success_rate": 0.2111111111111111,
+          "restore_max_error": 4.0978193283081055e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.3968580000028752,
+          "expert_success_rate": 0.8514851485148515,
+          "num_groups": 101,
+          "oracle_success_rate": 1.0,
+          "policy_expert_regret": 0.38333899815483846,
+          "policy_oracle_regret": 0.4754229130426256,
+          "policy_rollout_progress": 0.7721018394326219,
+          "policy_rollout_success_rate": 0.7524752475247525,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.51879838972042,
+          "expert_success_rate": 0.7666666666666667,
+          "num_groups": 90,
+          "oracle_success_rate": 0.9111111111111111,
+          "policy_expert_regret": 1.2014068081974982,
+          "policy_oracle_regret": 1.3635271353854073,
+          "policy_rollout_progress": 0.4004442221588559,
+          "policy_rollout_success_rate": 0.12222222222222222,
+          "restore_max_error": 4.76837158203125e-07
+        }
+      }
+    }
+  ]
+}

results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p375_safe_types_summary.md

@@ -0,0 +1,19 @@
+# h=16 Best-Policy Checkpoint Rollout
+
+Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
+Objective: `near_miss_policy_bc5`
+Result file: `policy_rollout_retrieval_residual_scale0p375_safe_types.json`
+Completed seeds: 3
+Baseline h=4 policy success: 29.67%
+Baseline h=16 rank-checkpoint success: 29.74%
+
+Mean success: 33.51% +/- 0.88%
+Gain vs h=16 rank checkpoint: +3.77%
+Mean progress: 55.40%
+Mean action MSE to best: 0.414
+
+| seed | mode | k | retrieval K | retrieval metric | min type success | residual scale | sigma | opt steps | trust | success | progress | oracle | action MSE |
+|---:|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
+| 0 | retrieval_residual | 16 | 1 | raw | 0.00 | 0.38 | 0.00 | 0 | 0.00 | 32.70% | 54.33% | 85.74% | 0.391 |
+| 1 | retrieval_residual | 16 | 1 | raw | 0.00 | 0.38 | 0.00 | 0 | 0.00 | 33.39% | 55.61% | 86.96% | 0.406 |
+| 2 | retrieval_residual | 16 | 1 | raw | 0.00 | 0.38 | 0.00 | 0 | 0.00 | 34.43% | 56.27% | 87.65% | 0.443 |

results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p40_safe_types_summary.json

@@ -0,0 +1,286 @@
+{
+  "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs",
+  "objective": "near_miss_policy_bc5",
+  "out_name": "policy_rollout_retrieval_residual_scale0p40_safe_types.json",
+  "num_completed": 3,
+  "baseline_h4_policy_success": 0.2967,
+  "baseline_h16_rank_checkpoint_success": 0.29739130434782607,
+  "mean_success": 0.3373913043478261,
+  "std_success": 0.007580693814853351,
+  "mean_progress": 0.5542008624550918,
+  "mean_action_mse_to_best": 0.41781920873492523,
+  "gain_vs_h4": 0.040691304347826085,
+  "gain_vs_h16_rank_checkpoint": 0.040000000000000036,
+  "rows": [
+    {
+      "seed": 0,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_0/policy_rollout_retrieval_residual_scale0p40_safe_types.json",
+      "num_groups": 575,
+      "selection_mode": "retrieval_residual",
+      "num_candidates": 16,
+      "candidate_sigma": 0.0,
+      "field_optim_steps": 0,
+      "field_optim_step_size": 0.0,
+      "field_optim_trust_radius": 0.0,
+      "field_optim_l2_penalty": 0.0,
+      "retrieval_neighbors": 1,
+      "retrieval_metric": "raw",
+      "retrieval_type_min_success": 0.0,
+      "retrieval_residual_scale": 0.4,
+      "policy_rollout_success_rate": 0.33217391304347826,
+      "policy_rollout_progress": 0.5451311302693237,
+      "oracle_success_rate": 0.8573913043478261,
+      "action_mse_to_best": 0.39242279884041004,
+      "best_policy_val": {
+        "bc_loss": 0.13721593966086706,
+        "field_effect_loss": 0.009290305380192068,
+        "field_potential_loss": 0.2666468388504452,
+        "field_preference_loss": 0.5130573478009965,
+        "lattice_edges": 3833.3333333333335,
+        "progress_mae": 0.1933159919248687,
+        "rank_acc": 0.8265031774838766,
+        "rank_loss": 0.5130523675017886,
+        "regret_mae": 0.3756548762321472,
+        "success_accuracy": 0.8773836526605818,
+        "total_loss": 1.5581054819954767
+      },
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.32044420050956385,
+          "expert_success_rate": 0.8865979381443299,
+          "num_groups": 97,
+          "oracle_success_rate": 0.9278350515463918,
+          "policy_expert_regret": 1.0792337135248578,
+          "policy_oracle_regret": 1.1056491912333006,
+          "policy_rollout_progress": 0.570563180084081,
+          "policy_rollout_success_rate": 0.2268041237113402,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.3117964074302178,
+          "expert_success_rate": 0.9375,
+          "num_groups": 208,
+          "oracle_success_rate": 0.9471153846153846,
+          "policy_expert_regret": 1.0220833119685546,
+          "policy_oracle_regret": 1.0288608400688435,
+          "policy_rollout_progress": 0.5906748250305939,
+          "policy_rollout_success_rate": 0.3125,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.6178919926479265,
+          "expert_success_rate": 0.19480519480519481,
+          "num_groups": 77,
+          "oracle_success_rate": 0.36363636363636365,
+          "policy_expert_regret": 0.19646811384447643,
+          "policy_oracle_regret": 0.3675295881422043,
+          "policy_rollout_progress": 0.3463196777118757,
+          "policy_rollout_success_rate": 0.24675324675324675,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.37073071645770933,
+          "expert_success_rate": 0.8617021276595744,
+          "num_groups": 94,
+          "oracle_success_rate": 0.9893617021276596,
+          "policy_expert_regret": 0.5719183905327574,
+          "policy_oracle_regret": 0.6687882122524241,
+          "policy_rollout_progress": 0.6822756175348099,
+          "policy_rollout_success_rate": 0.648936170212766,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.47757574098391664,
+          "expert_success_rate": 0.6767676767676768,
+          "num_groups": 99,
+          "oracle_success_rate": 0.8585858585858586,
+          "policy_expert_regret": 0.9663701350941802,
+          "policy_oracle_regret": 1.129547276280143,
+          "policy_rollout_progress": 0.44893822826520363,
+          "policy_rollout_success_rate": 0.24242424242424243,
+          "restore_max_error": 3.948807716369629e-07
+        }
+      }
+    },
+    {
+      "seed": 1,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_1/policy_rollout_retrieval_residual_scale0p40_safe_types.json",
+      "num_groups": 575,
+      "selection_mode": "retrieval_residual",
+      "num_candidates": 16,
+      "candidate_sigma": 0.0,
+      "field_optim_steps": 0,
+      "field_optim_step_size": 0.0,
+      "field_optim_trust_radius": 0.0,
+      "field_optim_l2_penalty": 0.0,
+      "retrieval_neighbors": 1,
+      "retrieval_metric": "raw",
+      "retrieval_type_min_success": 0.0,
+      "retrieval_residual_scale": 0.4,
+      "policy_rollout_success_rate": 0.3339130434782609,
+      "policy_rollout_progress": 0.5542254948337981,
+      "oracle_success_rate": 0.8695652173913043,
+      "action_mse_to_best": 0.411248748169645,
+      "best_policy_val": {
+        "bc_loss": 0.1315989200439718,
+        "field_effect_loss": 0.010110370814800262,
+        "field_potential_loss": 0.31546536915832096,
+        "field_preference_loss": 0.4960637440284093,
+        "lattice_edges": 3833.3333333333335,
+        "progress_mae": 0.19023226698239645,
+        "rank_acc": 0.8255469501018524,
+        "rank_loss": 0.4960531195004781,
+        "regret_mae": 0.42156532737943864,
+        "success_accuracy": 0.8807473679383596,
+        "total_loss": 1.561984618504842
+      },
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.33202078337657503,
+          "expert_success_rate": 0.8584070796460177,
+          "num_groups": 113,
+          "oracle_success_rate": 0.9380530973451328,
+          "policy_expert_regret": 1.0097657771764603,
+          "policy_oracle_regret": 1.09114876837857,
+          "policy_rollout_progress": 0.6173713228343862,
+          "policy_rollout_success_rate": 0.21238938053097345,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.35542732936775556,
+          "expert_success_rate": 0.9402173913043478,
+          "num_groups": 184,
+          "oracle_success_rate": 0.9456521739130435,
+          "policy_expert_regret": 1.0758233594343714,
+          "policy_oracle_regret": 1.0833602644017208,
+          "policy_rollout_progress": 0.56405944644433,
+          "policy_rollout_success_rate": 0.29891304347826086,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.5873942357536993,
+          "expert_success_rate": 0.25,
+          "num_groups": 76,
+          "oracle_success_rate": 0.40789473684210525,
+          "policy_expert_regret": 0.29673690845364736,
+          "policy_oracle_regret": 0.5491653543696821,
+          "policy_rollout_progress": 0.26781033527048864,
+          "policy_rollout_success_rate": 0.15789473684210525,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.3772244975642041,
+          "expert_success_rate": 0.8198198198198198,
+          "num_groups": 111,
+          "oracle_success_rate": 1.0,
+          "policy_expert_regret": 0.34765370606302143,
+          "policy_oracle_regret": 0.4142439628506566,
+          "policy_rollout_progress": 0.8019722533655597,
+          "policy_rollout_success_rate": 0.7837837837837838,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.5168919991804185,
+          "expert_success_rate": 0.6923076923076923,
+          "num_groups": 91,
+          "oracle_success_rate": 0.8571428571428571,
+          "policy_expert_regret": 1.1255206906206006,
+          "policy_oracle_regret": 1.2976684447500733,
+          "policy_rollout_progress": 0.39293688240942065,
+          "policy_rollout_success_rate": 0.15384615384615385,
+          "restore_max_error": 3.948807716369629e-07
+        }
+      }
+    },
+    {
+      "seed": 2,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs/near_miss_policy_bc5/seed_2/policy_rollout_retrieval_residual_scale0p40_safe_types.json",
+      "num_groups": 575,
+      "selection_mode": "retrieval_residual",
+      "num_candidates": 16,
+      "candidate_sigma": 0.0,
+      "field_optim_steps": 0,
+      "field_optim_step_size": 0.0,
+      "field_optim_trust_radius": 0.0,
+      "field_optim_l2_penalty": 0.0,
+      "retrieval_neighbors": 1,
+      "retrieval_metric": "raw",
+      "retrieval_type_min_success": 0.0,
+      "retrieval_residual_scale": 0.4,
+      "policy_rollout_success_rate": 0.34608695652173915,
+      "policy_rollout_progress": 0.5632459622621536,
+      "oracle_success_rate": 0.8765217391304347,
+      "action_mse_to_best": 0.4497860791947207,
+      "best_policy_val": {
+        "bc_loss": 0.11367896075050037,
+        "field_effect_loss": 0.009670218582161598,
+        "field_potential_loss": 0.2641640139950646,
+        "field_preference_loss": 0.5130490180518892,
+        "lattice_edges": 3833.3333333333335,
+        "progress_mae": 0.2021729110015763,
+        "rank_acc": 0.8333857821093665,
+        "rank_loss": 0.5130119257503085,
+        "regret_mae": 0.3958987047274907,
+        "success_accuracy": 0.8680730561415354,
+        "total_loss": 1.4394984311527677
+      },
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.3585303530320137,
+          "expert_success_rate": 0.8229166666666666,
+          "num_groups": 96,
+          "oracle_success_rate": 0.9270833333333334,
+          "policy_expert_regret": 0.8652257267385721,
+          "policy_oracle_regret": 0.9582381844520569,
+          "policy_rollout_progress": 0.6420370054741701,
+          "policy_rollout_success_rate": 0.3333333333333333,
+          "restore_max_error": 3.5762786865234375e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.37847183247783567,
+          "expert_success_rate": 0.9444444444444444,
+          "num_groups": 198,
+          "oracle_success_rate": 0.9595959595959596,
+          "policy_expert_regret": 1.0410469274384657,
+          "policy_oracle_regret": 1.048130377967174,
+          "policy_rollout_progress": 0.6015997416236334,
+          "policy_rollout_success_rate": 0.29797979797979796,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.6824962091114786,
+          "expert_success_rate": 0.24444444444444444,
+          "num_groups": 90,
+          "oracle_success_rate": 0.4666666666666667,
+          "policy_expert_regret": 0.3191297983750701,
+          "policy_oracle_regret": 0.5443947612618406,
+          "policy_rollout_progress": 0.3162850609877043,
+          "policy_rollout_success_rate": 0.2111111111111111,
+          "restore_max_error": 4.0978193283081055e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.40046302893079155,
+          "expert_success_rate": 0.8514851485148515,
+          "num_groups": 101,
+          "oracle_success_rate": 1.0,
+          "policy_expert_regret": 0.3628855461236274,
+          "policy_oracle_regret": 0.4364342873934472,
+          "policy_rollout_progress": 0.7912884848837806,
+          "policy_rollout_success_rate": 0.7722772277227723,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.5266581563692954,
+          "expert_success_rate": 0.7666666666666667,
+          "num_groups": 90,
+          "oracle_success_rate": 0.9111111111111111,
+          "policy_expert_regret": 1.2119419298238225,
+          "policy_oracle_regret": 1.3739223264985614,
+          "policy_rollout_progress": 0.38587038301759297,
+          "policy_rollout_success_rate": 0.12222222222222222,
+          "restore_max_error": 4.76837158203125e-07
+        }
+      }
+    }
+  ]
+}

results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_scale0p40_safe_types_summary.md

@@ -0,0 +1,19 @@
+# h=16 Best-Policy Checkpoint Rollout
+
+Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
+Objective: `near_miss_policy_bc5`
+Result file: `policy_rollout_retrieval_residual_scale0p40_safe_types.json`
+Completed seeds: 3
+Baseline h=4 policy success: 29.67%
+Baseline h=16 rank-checkpoint success: 29.74%
+
+Mean success: 33.74% +/- 0.76%
+Gain vs h=16 rank checkpoint: +4.00%
+Mean progress: 55.42%
+Mean action MSE to best: 0.418
+
+| seed | mode | k | retrieval K | retrieval metric | min type success | residual scale | sigma | opt steps | trust | success | progress | oracle | action MSE |
+|---:|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
+| 0 | retrieval_residual | 16 | 1 | raw | 0.00 | 0.40 | 0.00 | 0 | 0.00 | 33.22% | 54.51% | 85.74% | 0.392 |
+| 1 | retrieval_residual | 16 | 1 | raw | 0.00 | 0.40 | 0.00 | 0 | 0.00 | 33.39% | 55.42% | 86.96% | 0.411 |
+| 2 | retrieval_residual | 16 | 1 | raw | 0.00 | 0.40 | 0.00 | 0 | 0.00 | 34.61% | 56.32% | 87.65% | 0.450 |

results/paper_core_results.md

@@ -27,7 +27,8 @@ baseline is the h=16 rank-checkpoint online rollout (`29.74%`).
 | Train-state residual retrieval, scale 0.50 | No | No | 33.33% | +3.59 pp | Calibrated local tangent transport |
 | Train-state residual retrieval, no random residuals | No | No | 33.45% | +3.71 pp | Removing anti-goal random residuals helps slightly |
 | Train-state residual retrieval, no random/wrong-direction residuals | No | No | 33.57% | +3.83 pp | Anti-goal family masking improves the clean bridge |
-| Train-state residual retrieval, policy/no-op/wrong-gripper residuals | No | No | 33.68% | +3.94 pp | Current best deployment-clean diagnostic |
+| Train-state residual retrieval, policy/no-op/wrong-gripper residuals | No | No | 33.68% | +3.94 pp | Typed family mask improves clean bridge |
+| Train-state residual retrieval, policy/no-op/wrong-gripper, scale 0.35 | No | No | 33.74% | +4.00 pp | Current best deployment-clean diagnostic |
 | Train-state residual retrieval, z-score metric | No | No | 32.23% | +2.49 pp | State normalization hurts nearest tangent retrieval here |
 | Train-state residual retrieval, z-score metric + anti-goal mask | No | No | 32.75% | +3.01 pp | Masking helps z-score but remains below raw |
 | Train-state residual retrieval, train family reliability prior | No | No | 33.33% | +3.59 pp | Train terminal-success thresholds through 0.75 do not filter enough |
@@ -54,7 +55,7 @@ Suggested main-table rows:
 7. Field-selected no-expert policy + field, seed-0 train map
 8. Field-selected no-expert policy + field, aligned allmap
 9. Train-state residual retrieval, scale 0.50
-10. Train-state residual retrieval, typed safe families
+10. Train-state residual retrieval, typed safe families at scale 0.35
 11. Residual+Gaussian hybrid, K32 sigma0.35
 12. Lattice, near-miss only
 13. Lattice, no expert

results/paper_story_memo.md

@@ -16,7 +16,7 @@ when queried on proposal geometry that matches those local counterfactuals.
 | Same-state local counterfactual proposals are the mechanism | near-miss-only lattice is 55.94%; removing expert+near_miss drops to 25.57% | Strongly supported |
 | Conservative same-state result is large | no-expert lattice is 56.99% vs 29.74% policy | Main result |
 | Full lattice gives upper result | full lattice is 69.33%, oracle is 86.78% | Strong but label expert proposal clearly |
-| Deployment-clean proposal is currently a bottleneck | best clean residual transport is 33.68%, far below 56.99% | Supported |
+| Deployment-clean proposal is currently a bottleneck | best clean residual transport is 33.74%, far below 56.99% | Supported |
 | Gradient-based field optimization does not solve the clean proposal gap | `field_optim` best observed result is 25.39% | Negative diagnostic |
 | A broader non-expert proposal target does not reduce the proposal gap | direct broad non-expert policy is 27.88%; with field scoring it is 26.49% | Negative diagnostic |
 | Counterfactual residuals transfer better than absolute retrieved actions | nearest residual retrieval is 32.12% vs absolute retrieval 28.93%; KNN4 residual drops to 29.91% | Supported as a clean bridge |
@@ -24,7 +24,7 @@ when queried on proposal geometry that matches those local counterfactuals.
 | Residual transport and Gaussian local proposals are not complementary here | hybrid K32/K64 reach 31.30%/30.90%, below residual-only | Negative diagnostic |
 | Seed-0 train-split field-teacher distillation does not solve the proposal gap | direct student is 26.84%; with field scoring it is 27.65% | Negative diagnostic |
 | All-split field-teacher distillation does not fix checkpointing/coverage | allmap direct is 28.00%; field-guided best is 26.49% despite 100% target coverage | Negative diagnostic |
-| Residual family consistency improves clean transport | policy/no-op/wrong-gripper typed residuals reach 33.68%, above raw 33.33% | Supported as diagnostic |
+| Residual family consistency improves clean transport | policy/no-op/wrong-gripper typed residuals reach 33.74%, above raw 33.33% | Supported as diagnostic |
 | Z-score retrieval metric does not help | z-score rows reach 32.23-32.81%, below raw retrieval | Negative diagnostic |
 | Train-split residual family reliability does not recover the typed mask | thresholds through 0.75 do not filter the bad families; rows stay at 33.33% | Negative diagnostic |
 
@@ -44,7 +44,7 @@ clean proposal result, the intended main rows are:
 8. Field-selected no-expert proposal + field, aligned allmap: 26.49%
 9. Train-state residual retrieval: 32.12%
 10. Train-state residual retrieval, scale 0.50: 33.33%
-11. Train-state residual retrieval, typed safe families: 33.68%
+11. Train-state residual retrieval, typed safe families: 33.74%
 12. Z-score residual retrieval: 32.23-32.81%
 13. Train-family reliability prior: 32.93-33.33%
 14. Residual+Gaussian hybrid K32/K64: 31.30% / 30.90%
@@ -103,12 +103,15 @@ Last checked: `2026-06-28 06:24 UTC`. No DoVLA jobs are currently queued.
   Thresholds `0.10`, `0.25`, `0.50`, and `0.75` do not filter the bad residual
   families and remain at the raw scale-0.50 result (33.33%), except scale-0.25
   threshold `0.25` at 32.93%.
+- `14859503`-`14859597`: completed typed-safe residual scale fine/zoom sweep.
+  Scales `0.325`, `0.35`, and `0.40` tie as the best clean rows at 33.74%;
+  scales above `0.50` fall back.
 
 ## Decision Notes
 
 - Promote same-state no-expert lattice (56.99%) as the conservative mechanism
   result.
-- Use typed safe residual transport (33.68%) only as the current best clean
+- Use typed safe residual transport (33.74%) only as the current best clean
   deployment diagnostic, not as a SOTA claim.
 - Treat z-score retrieval, train-family reliability priors, Gaussian hybrids,
   field optimization, and field-teacher distillation as negative diagnostics

results/paper_table_status.json

@@ -451,14 +451,14 @@
       "story_role": "typed tangent scale zoom sweep",
       "fallback_success": null,
       "pending_job": "14859589/14859590",
-      "path_exists": false,
-      "status": "pending",
-      "success": null,
-      "std_success": null,
+      "path_exists": true,
+      "status": "complete",
+      "success": 0.33507246376811595,
+      "std_success": 0.005020437123388053,
       "completed_seeds": null,
-      "num_completed": null,
+      "num_completed": 3,
       "best_config": null,
-      "gain_vs_h16_policy": null
+      "gain_vs_h16_policy": 0.03768115942028988
     },
     {
       "key": "retrieval_residual_scale0325_safe_types",
@@ -470,14 +470,14 @@
       "story_role": "typed tangent scale zoom sweep",
       "fallback_success": null,
       "pending_job": "14859591/14859592",
-      "path_exists": false,
-      "status": "pending",
-      "success": null,
-      "std_success": null,
+      "path_exists": true,
+      "status": "complete",
+      "success": 0.3373913043478261,
+      "std_success": 0.006956521739130445,
       "completed_seeds": null,
-      "num_completed": null,
+      "num_completed": 3,
       "best_config": null,
-      "gain_vs_h16_policy": null
+      "gain_vs_h16_policy": 0.040000000000000036
     },
     {
       "key": "retrieval_residual_scale0375_safe_types",
@@ -489,14 +489,14 @@
       "story_role": "typed tangent scale zoom sweep",
       "fallback_success": null,
       "pending_job": "14859593/14859594",
-      "path_exists": false,
-      "status": "pending",
-      "success": null,
-      "std_success": null,
+      "path_exists": true,
+      "status": "complete",
+      "success": 0.33507246376811595,
+      "std_success": 0.008753431229299422,
       "completed_seeds": null,
-      "num_completed": null,
+      "num_completed": 3,
       "best_config": null,
-      "gain_vs_h16_policy": null
+      "gain_vs_h16_policy": 0.03768115942028988
     },
     {
       "key": "retrieval_residual_scale040_safe_types",
@@ -508,14 +508,14 @@
       "story_role": "typed tangent scale zoom sweep",
       "fallback_success": null,
       "pending_job": "14859595/14859596",
-      "path_exists": false,
-      "status": "pending",
-      "success": null,
-      "std_success": null,
+      "path_exists": true,
+      "status": "complete",
+      "success": 0.3373913043478261,
+      "std_success": 0.007580693814853351,
       "completed_seeds": null,
-      "num_completed": null,
+      "num_completed": 3,
       "best_config": null,
-      "gain_vs_h16_policy": null
+      "gain_vs_h16_policy": 0.040000000000000036
     },
     {
       "key": "retrieval_residual_scale045_safe_types",

results/paper_table_status.md

@@ -26,10 +26,10 @@ Baseline h=16 policy: 29.74%
 | retrieval_residual_scale025_no_random_wrongdir | Train-state residual retrieval, scale 0.25, no random/wrong-direction residuals | complete | 33.45% | +3.71 pp | yes | no | no | anti-goal residual family mask ablation |
 | retrieval_residual_scale050_safe_types | Train-state residual retrieval, scale 0.50, policy/no-op/wrong-gripper residuals | complete | 33.68% | +3.94 pp | yes | no | no | typed tangent-family mask ablation |
 | retrieval_residual_scale035_safe_types | Train-state residual retrieval, scale 0.35, policy/no-op/wrong-gripper residuals | complete | 33.74% | +4.00 pp | yes | no | no | typed tangent scale fine sweep |
-| retrieval_residual_scale030_safe_types | Train-state residual retrieval, scale 0.30, policy/no-op/wrong-gripper residuals | pending 14859589/14859590 | pending | pending | yes | no | no | typed tangent scale zoom sweep |
-| retrieval_residual_scale0325_safe_types | Train-state residual retrieval, scale 0.325, policy/no-op/wrong-gripper residuals | pending 14859591/14859592 | pending | pending | yes | no | no | typed tangent scale zoom sweep |
-| retrieval_residual_scale0375_safe_types | Train-state residual retrieval, scale 0.375, policy/no-op/wrong-gripper residuals | pending 14859593/14859594 | pending | pending | yes | no | no | typed tangent scale zoom sweep |
-| retrieval_residual_scale040_safe_types | Train-state residual retrieval, scale 0.40, policy/no-op/wrong-gripper residuals | pending 14859595/14859596 | pending | pending | yes | no | no | typed tangent scale zoom sweep |
+| retrieval_residual_scale030_safe_types | Train-state residual retrieval, scale 0.30, policy/no-op/wrong-gripper residuals | complete | 33.51% | +3.77 pp | yes | no | no | typed tangent scale zoom sweep |
+| retrieval_residual_scale0325_safe_types | Train-state residual retrieval, scale 0.325, policy/no-op/wrong-gripper residuals | complete | 33.74% | +4.00 pp | yes | no | no | typed tangent scale zoom sweep |
+| retrieval_residual_scale0375_safe_types | Train-state residual retrieval, scale 0.375, policy/no-op/wrong-gripper residuals | complete | 33.51% | +3.77 pp | yes | no | no | typed tangent scale zoom sweep |
+| retrieval_residual_scale040_safe_types | Train-state residual retrieval, scale 0.40, policy/no-op/wrong-gripper residuals | complete | 33.74% | +4.00 pp | yes | no | no | typed tangent scale zoom sweep |
 | retrieval_residual_scale045_safe_types | Train-state residual retrieval, scale 0.45, policy/no-op/wrong-gripper residuals | complete | 33.51% | +3.77 pp | yes | no | no | typed tangent scale fine sweep |
 | retrieval_residual_scale060_safe_types | Train-state residual retrieval, scale 0.60, policy/no-op/wrong-gripper residuals | complete | 33.39% | +3.65 pp | yes | no | no | typed tangent scale fine sweep |
 | retrieval_residual_scale070_safe_types | Train-state residual retrieval, scale 0.70, policy/no-op/wrong-gripper residuals | complete | 33.16% | +3.42 pp | yes | no | no | typed tangent scale fine sweep |