Auto-sync: 2026-06-27 10:32:18

results/h16_lattice_no_near_miss_no_expert_summary.json

@@ -0,0 +1,227 @@
+{
+  "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs",
+  "out_name": "lattice_no_near_miss_no_expert_rollout.json",
+  "num_completed": 3,
+  "baseline_h4_policy_success": 0.2967,
+  "baseline_h16_policy_success": 0.29739130434782607,
+  "mean_success": 0.5698550724637681,
+  "std_success": 0.046188021535170105,
+  "mean_progress": 0.7501174603224449,
+  "mean_oracle_success": 0.8678260869565217,
+  "mean_action_mse_to_best": 0.4590102900127523,
+  "gain_vs_h4": 0.2731550724637681,
+  "gain_vs_h16_policy": 0.27246376811594203,
+  "selected_candidate_type_counts": {
+    "lattice_near_miss": 1263,
+    "lattice_no_op": 222,
+    "lattice_random_negative": 144,
+    "lattice_wrong_direction": 34,
+    "lattice_wrong_gripper": 62
+  },
+  "rows": [
+    {
+      "seed": 0,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_0/lattice_no_near_miss_no_expert_rollout.json",
+      "selection_mode": "lattice",
+      "num_candidates": 16,
+      "num_groups": 575,
+      "policy_rollout_success_rate": 0.5165217391304348,
+      "policy_rollout_progress": 0.7308024795159049,
+      "oracle_success_rate": 0.8573913043478261,
+      "action_mse_to_best": 0.4291479373977864,
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.2888141949958115,
+          "expert_success_rate": 0.8865979381443299,
+          "num_groups": 97,
+          "oracle_success_rate": 0.9278350515463918,
+          "policy_expert_regret": 0.45705366580142187,
+          "policy_oracle_regret": 0.47029784183526774,
+          "policy_rollout_progress": 0.8553990655645882,
+          "policy_rollout_success_rate": 0.5773195876288659,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.4340394471700375,
+          "expert_success_rate": 0.9375,
+          "num_groups": 208,
+          "oracle_success_rate": 0.9471153846153846,
+          "policy_expert_regret": 0.6624748194769311,
+          "policy_oracle_regret": 0.6691413935309706,
+          "policy_rollout_progress": 0.7513381671697761,
+          "policy_rollout_success_rate": 0.5048076923076923,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.5183976920587676,
+          "expert_success_rate": 0.19480519480519481,
+          "num_groups": 77,
+          "oracle_success_rate": 0.36363636363636365,
+          "policy_expert_regret": 0.1787171581058533,
+          "policy_oracle_regret": 0.37732240485100005,
+          "policy_rollout_progress": 0.3172257935942768,
+          "policy_rollout_success_rate": 0.16883116883116883,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.39197504613300826,
+          "expert_success_rate": 0.8617021276595744,
+          "num_groups": 94,
+          "oracle_success_rate": 0.9893617021276596,
+          "policy_expert_regret": 0.2812512713543912,
+          "policy_oracle_regret": 0.3391457902307206,
+          "policy_rollout_progress": 0.8417052735990667,
+          "policy_rollout_success_rate": 0.8191489361702128,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.5222485399266441,
+          "expert_success_rate": 0.6767676767676768,
+          "num_groups": 99,
+          "oracle_success_rate": 0.8585858585858586,
+          "policy_expert_regret": 0.5384464679342328,
+          "policy_oracle_regret": 0.574341721606977,
+          "policy_rollout_progress": 0.7819464218736899,
+          "policy_rollout_success_rate": 0.46464646464646464,
+          "restore_max_error": 3.948807716369629e-07
+        }
+      }
+    },
+    {
+      "seed": 1,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_1/lattice_no_near_miss_no_expert_rollout.json",
+      "selection_mode": "lattice",
+      "num_candidates": 16,
+      "num_groups": 575,
+      "policy_rollout_success_rate": 0.5965217391304348,
+      "policy_rollout_progress": 0.756582982868928,
+      "oracle_success_rate": 0.8695652173913043,
+      "action_mse_to_best": 0.44271026729079693,
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.3169037180301626,
+          "expert_success_rate": 0.8584070796460177,
+          "num_groups": 113,
+          "oracle_success_rate": 0.9380530973451328,
+          "policy_expert_regret": 0.39746501211044005,
+          "policy_oracle_regret": 0.43380374942733124,
+          "policy_rollout_progress": 0.858787138245802,
+          "policy_rollout_success_rate": 0.6283185840707964,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.4862241648218554,
+          "expert_success_rate": 0.9402173913043478,
+          "num_groups": 184,
+          "oracle_success_rate": 0.9456521739130435,
+          "policy_expert_regret": 0.4408608768540232,
+          "policy_oracle_regret": 0.44088488132895337,
+          "policy_rollout_progress": 0.8249527744381971,
+          "policy_rollout_success_rate": 0.657608695652174,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.5220103718977618,
+          "expert_success_rate": 0.25,
+          "num_groups": 76,
+          "oracle_success_rate": 0.40789473684210525,
+          "policy_expert_regret": 0.2234261535524734,
+          "policy_oracle_regret": 0.44941299718706623,
+          "policy_rollout_progress": 0.32774039636071967,
+          "policy_rollout_success_rate": 0.17105263157894737,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.35074781935225735,
+          "expert_success_rate": 0.8198198198198198,
+          "num_groups": 111,
+          "oracle_success_rate": 1.0,
+          "policy_expert_regret": 0.24994636481409674,
+          "policy_oracle_regret": 0.3667086133280316,
+          "policy_rollout_progress": 0.8314895848701666,
+          "policy_rollout_success_rate": 0.8018018018018018,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.5568927584093187,
+          "expert_success_rate": 0.6923076923076923,
+          "num_groups": 91,
+          "oracle_success_rate": 0.8571428571428571,
+          "policy_expert_regret": 0.47163032846791403,
+          "policy_oracle_regret": 0.5352666910205569,
+          "policy_rollout_progress": 0.7582125712882032,
+          "policy_rollout_success_rate": 0.5384615384615384,
+          "restore_max_error": 3.948807716369629e-07
+        }
+      }
+    },
+    {
+      "seed": 2,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_2/lattice_no_near_miss_no_expert_rollout.json",
+      "selection_mode": "lattice",
+      "num_candidates": 16,
+      "num_groups": 575,
+      "policy_rollout_success_rate": 0.5965217391304348,
+      "policy_rollout_progress": 0.7629669185825017,
+      "oracle_success_rate": 0.8765217391304347,
+      "action_mse_to_best": 0.5051726653496735,
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.3387383111627666,
+          "expert_success_rate": 0.8229166666666666,
+          "num_groups": 96,
+          "oracle_success_rate": 0.9270833333333334,
+          "policy_expert_regret": 0.349092789615194,
+          "policy_oracle_regret": 0.38970588706433773,
+          "policy_rollout_progress": 0.8847468371192614,
+          "policy_rollout_success_rate": 0.6458333333333334,
+          "restore_max_error": 3.5762786865234375e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.5211711752068515,
+          "expert_success_rate": 0.9444444444444444,
+          "num_groups": 198,
+          "oracle_success_rate": 0.9595959595959596,
+          "policy_expert_regret": 0.38791667779136185,
+          "policy_oracle_regret": 0.38796894105546403,
+          "policy_rollout_progress": 0.8505876985086938,
+          "policy_rollout_success_rate": 0.702020202020202,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.682747141416702,
+          "expert_success_rate": 0.24444444444444444,
+          "num_groups": 90,
+          "oracle_success_rate": 0.4666666666666667,
+          "policy_expert_regret": 0.2159867504404651,
+          "policy_oracle_regret": 0.4278881938921081,
+          "policy_rollout_progress": 0.38586775238315263,
+          "policy_rollout_success_rate": 0.25555555555555554,
+          "restore_max_error": 4.0978193283081055e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.3426748497024326,
+          "expert_success_rate": 0.8514851485148515,
+          "num_groups": 101,
+          "oracle_success_rate": 1.0,
+          "policy_expert_regret": 0.33127955694009764,
+          "policy_oracle_regret": 0.40476624254543003,
+          "policy_rollout_progress": 0.8130555396327878,
+          "policy_rollout_success_rate": 0.7821782178217822,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.6522901051781244,
+          "expert_success_rate": 0.7666666666666667,
+          "num_groups": 90,
+          "oracle_success_rate": 0.9111111111111111,
+          "policy_expert_regret": 0.6501921941836675,
+          "policy_oracle_regret": 0.6890770451890098,
+          "policy_rollout_progress": 0.7611912255485852,
+          "policy_rollout_success_rate": 0.4444444444444444,
+          "restore_max_error": 4.76837158203125e-07
+        }
+      }
+    }
+  ]
+}

results/h16_lattice_no_near_miss_no_expert_summary.md

@@ -0,0 +1,24 @@
+# h=16 Lattice-Selected Rollout
+
+Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs`
+Completed seeds: 3
+Baseline h=4 policy success: 29.67%
+Baseline h=16 policy success: 29.74%
+
+Mean success: 56.99% +/- 4.62%
+Gain vs h=16 policy: +27.25%
+Mean oracle success: 86.78%
+Mean progress: 75.01%
+
+| seed | success | progress | oracle | candidates | action MSE |
+|---:|---:|---:|---:|---:|---:|
+| 0 | 51.65% | 73.08% | 85.74% | 16 | 0.429 |
+| 1 | 59.65% | 75.66% | 86.96% | 16 | 0.443 |
+| 2 | 59.65% | 76.30% | 87.65% | 16 | 0.505 |
+
+Selected candidate types:
+- lattice_near_miss: 1263
+- lattice_no_op: 222
+- lattice_random_negative: 144
+- lattice_wrong_direction: 34
+- lattice_wrong_gripper: 62

results/h16_retrieval_lattice_no_expert_summary.json

@@ -0,0 +1,227 @@
+{
+  "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs",
+  "out_name": "retrieval_lattice_no_expert_rollout.json",
+  "num_completed": 3,
+  "baseline_h4_policy_success": 0.2967,
+  "baseline_h16_policy_success": 0.29739130434782607,
+  "mean_success": 0.271304347826087,
+  "std_success": 0.018157054798105306,
+  "mean_progress": 0.5005514273950088,
+  "mean_oracle_success": 0.8678260869565217,
+  "mean_action_mse_to_best": 0.5836967330339594,
+  "gain_vs_h4": -0.02539565217391304,
+  "gain_vs_h16_policy": -0.02608695652173909,
+  "selected_candidate_type_counts": {
+    "lattice_near_miss": 1276,
+    "lattice_no_op": 212,
+    "lattice_random_negative": 143,
+    "lattice_wrong_direction": 27,
+    "lattice_wrong_gripper": 67
+  },
+  "rows": [
+    {
+      "seed": 0,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_0/retrieval_lattice_no_expert_rollout.json",
+      "selection_mode": "retrieval_lattice",
+      "num_candidates": 16,
+      "num_groups": 575,
+      "policy_rollout_success_rate": 0.25043478260869567,
+      "policy_rollout_progress": 0.49867984499296414,
+      "oracle_success_rate": 0.8573913043478261,
+      "action_mse_to_best": 0.5693303860076095,
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.35727074150879357,
+          "expert_success_rate": 0.8865979381443299,
+          "num_groups": 97,
+          "oracle_success_rate": 0.9278350515463918,
+          "policy_expert_regret": 1.1045561598748277,
+          "policy_oracle_regret": 1.1440839441781192,
+          "policy_rollout_progress": 0.5836748188918399,
+          "policy_rollout_success_rate": 0.17525773195876287,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.4945910416650944,
+          "expert_success_rate": 0.9375,
+          "num_groups": 208,
+          "oracle_success_rate": 0.9471153846153846,
+          "policy_expert_regret": 1.1998865354958421,
+          "policy_oracle_regret": 1.2065531095498814,
+          "policy_rollout_progress": 0.5120033742277883,
+          "policy_rollout_success_rate": 0.20673076923076922,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.8415415789206306,
+          "expert_success_rate": 0.19480519480519481,
+          "num_groups": 77,
+          "oracle_success_rate": 0.36363636363636365,
+          "policy_expert_regret": 0.35076726408215697,
+          "policy_oracle_regret": 0.5829326225589816,
+          "policy_rollout_progress": 0.23609928142085865,
+          "policy_rollout_success_rate": 0.09090909090909091,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.5355885234919,
+          "expert_success_rate": 0.8617021276595744,
+          "num_groups": 94,
+          "oracle_success_rate": 0.9893617021276596,
+          "policy_expert_regret": 0.5391762771822036,
+          "policy_oracle_regret": 0.6366645524159391,
+          "policy_rollout_progress": 0.6841333678111117,
+          "policy_rollout_success_rate": 0.6595744680851063,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.7544520260258154,
+          "expert_success_rate": 0.6767676767676768,
+          "num_groups": 99,
+          "oracle_success_rate": 0.8585858585858586,
+          "policy_expert_regret": 1.0733418645280781,
+          "policy_oracle_regret": 1.2771704949513831,
+          "policy_rollout_progress": 0.41555121571126613,
+          "policy_rollout_success_rate": 0.15151515151515152,
+          "restore_max_error": 3.948807716369629e-07
+        }
+      }
+    },
+    {
+      "seed": 1,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_1/retrieval_lattice_no_expert_rollout.json",
+      "selection_mode": "retrieval_lattice",
+      "num_candidates": 16,
+      "num_groups": 575,
+      "policy_rollout_success_rate": 0.28,
+      "policy_rollout_progress": 0.49805013884861105,
+      "oracle_success_rate": 0.8695652173913043,
+      "action_mse_to_best": 0.5685278348916251,
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.3830389907435242,
+          "expert_success_rate": 0.8584070796460177,
+          "num_groups": 113,
+          "oracle_success_rate": 0.9380530973451328,
+          "policy_expert_regret": 0.9959914846494134,
+          "policy_oracle_regret": 1.0887631516804737,
+          "policy_rollout_progress": 0.6071000562041207,
+          "policy_rollout_success_rate": 0.24778761061946902,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.5331694724888582,
+          "expert_success_rate": 0.9402173913043478,
+          "num_groups": 184,
+          "oracle_success_rate": 0.9456521739130435,
+          "policy_expert_regret": 1.203628732283782,
+          "policy_oracle_regret": 1.2111656372511315,
+          "policy_rollout_progress": 0.5057589750377582,
+          "policy_rollout_success_rate": 0.20652173913043478,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.8114424096665492,
+          "expert_success_rate": 0.25,
+          "num_groups": 76,
+          "oracle_success_rate": 0.40789473684210525,
+          "policy_expert_regret": 0.3376943706940725,
+          "policy_oracle_regret": 0.577079668853242,
+          "policy_rollout_progress": 0.2529444615931042,
+          "policy_rollout_success_rate": 0.11842105263157894,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.4605464878248739,
+          "expert_success_rate": 0.8198198198198198,
+          "num_groups": 111,
+          "oracle_success_rate": 1.0,
+          "policy_expert_regret": 0.5753093846477904,
+          "policy_oracle_regret": 0.6765443254430015,
+          "policy_rollout_progress": 0.6748070259083498,
+          "policy_rollout_success_rate": 0.6486486486486487,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.7991937679944785,
+          "expert_success_rate": 0.6923076923076923,
+          "num_groups": 91,
+          "oracle_success_rate": 0.8571428571428571,
+          "policy_expert_regret": 1.1693862085486506,
+          "policy_oracle_regret": 1.3293362507125834,
+          "policy_rollout_progress": 0.3361484958575322,
+          "policy_rollout_success_rate": 0.15384615384615385,
+          "restore_max_error": 3.948807716369629e-07
+        }
+      }
+    },
+    {
+      "seed": 2,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_2/retrieval_lattice_no_expert_rollout.json",
+      "selection_mode": "retrieval_lattice",
+      "num_candidates": 16,
+      "num_groups": 575,
+      "policy_rollout_success_rate": 0.28347826086956524,
+      "policy_rollout_progress": 0.5049242983434511,
+      "oracle_success_rate": 0.8765217391304347,
+      "action_mse_to_best": 0.6132319782026436,
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.3866134468504849,
+          "expert_success_rate": 0.8229166666666666,
+          "num_groups": 96,
+          "oracle_success_rate": 0.9270833333333334,
+          "policy_expert_regret": 0.9153858826806148,
+          "policy_oracle_regret": 1.0359393774221342,
+          "policy_rollout_progress": 0.6372524791707596,
+          "policy_rollout_success_rate": 0.2604166666666667,
+          "restore_max_error": 3.5762786865234375e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.5512155196582428,
+          "expert_success_rate": 0.9444444444444444,
+          "num_groups": 198,
+          "oracle_success_rate": 0.9595959595959596,
+          "policy_expert_regret": 1.1322207739341514,
+          "policy_oracle_regret": 1.13930422446286,
+          "policy_rollout_progress": 0.5335958494447323,
+          "policy_rollout_success_rate": 0.2676767676767677,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.8858442465671235,
+          "expert_success_rate": 0.24444444444444444,
+          "num_groups": 90,
+          "oracle_success_rate": 0.4666666666666667,
+          "policy_expert_regret": 0.30565778343006966,
+          "policy_oracle_regret": 0.5796548407421344,
+          "policy_rollout_progress": 0.3154497137189739,
+          "policy_rollout_success_rate": 0.17777777777777778,
+          "restore_max_error": 4.0978193283081055e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.46022101115472247,
+          "expert_success_rate": 0.8514851485148515,
+          "num_groups": 101,
+          "oracle_success_rate": 1.0,
+          "policy_expert_regret": 0.6615248527562264,
+          "policy_oracle_regret": 0.7336106011182955,
+          "policy_rollout_progress": 0.6525280127430907,
+          "policy_rollout_success_rate": 0.6138613861386139,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.8904946595430374,
+          "expert_success_rate": 0.7666666666666667,
+          "num_groups": 90,
+          "oracle_success_rate": 0.9111111111111111,
+          "policy_expert_regret": 1.3177169235216246,
+          "policy_oracle_regret": 1.4797099113464356,
+          "policy_rollout_progress": 0.32452724261416327,
+          "policy_rollout_success_rate": 0.07777777777777778,
+          "restore_max_error": 4.76837158203125e-07
+        }
+      }
+    }
+  ]
+}

results/h16_retrieval_lattice_no_expert_summary.md

@@ -0,0 +1,24 @@
+# h=16 Lattice-Selected Rollout
+
+Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs`
+Completed seeds: 3
+Baseline h=4 policy success: 29.67%
+Baseline h=16 policy success: 29.74%
+
+Mean success: 27.13% +/- 1.82%
+Gain vs h=16 policy: -2.61%
+Mean oracle success: 86.78%
+Mean progress: 50.06%
+
+| seed | success | progress | oracle | candidates | action MSE |
+|---:|---:|---:|---:|---:|---:|
+| 0 | 25.04% | 49.87% | 85.74% | 16 | 0.569 |
+| 1 | 28.00% | 49.81% | 86.96% | 16 | 0.569 |
+| 2 | 28.35% | 50.49% | 87.65% | 16 | 0.613 |
+
+Selected candidate types:
+- lattice_near_miss: 1276
+- lattice_no_op: 212
+- lattice_random_negative: 143
+- lattice_wrong_direction: 27
+- lattice_wrong_gripper: 67

results/h16_retrieval_lattice_summary.json

@@ -0,0 +1,228 @@
+{
+  "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs",
+  "out_name": "retrieval_lattice_rollout.json",
+  "num_completed": 3,
+  "baseline_h4_policy_success": 0.2967,
+  "baseline_h16_policy_success": 0.29739130434782607,
+  "mean_success": 0.28927536231884055,
+  "std_success": 0.007842173483170244,
+  "mean_progress": 0.5070036883118725,
+  "mean_oracle_success": 0.8678260869565217,
+  "mean_action_mse_to_best": 0.5642791039512857,
+  "gain_vs_h4": -0.007424637681159474,
+  "gain_vs_h16_policy": -0.008115942028985523,
+  "selected_candidate_type_counts": {
+    "lattice_expert": 970,
+    "lattice_near_miss": 361,
+    "lattice_no_op": 172,
+    "lattice_random_negative": 133,
+    "lattice_wrong_direction": 25,
+    "lattice_wrong_gripper": 64
+  },
+  "rows": [
+    {
+      "seed": 0,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_0/retrieval_lattice_rollout.json",
+      "selection_mode": "retrieval_lattice",
+      "num_candidates": 16,
+      "num_groups": 575,
+      "policy_rollout_success_rate": 0.2817391304347826,
+      "policy_rollout_progress": 0.5097964013108741,
+      "oracle_success_rate": 0.8573913043478261,
+      "action_mse_to_best": 0.5467841655761003,
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.35563730830614715,
+          "expert_success_rate": 0.8865979381443299,
+          "num_groups": 97,
+          "oracle_success_rate": 0.9278350515463918,
+          "policy_expert_regret": 1.0326172715609836,
+          "policy_oracle_regret": 1.0722317271625872,
+          "policy_rollout_progress": 0.6039806441547945,
+          "policy_rollout_success_rate": 0.2268041237113402,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.4484794621821493,
+          "expert_success_rate": 0.9375,
+          "num_groups": 208,
+          "oracle_success_rate": 0.9471153846153846,
+          "policy_expert_regret": 1.1537728436658374,
+          "policy_oracle_regret": 1.1537940783936602,
+          "policy_rollout_progress": 0.5311086406134284,
+          "policy_rollout_success_rate": 0.2403846153846154,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.84151822512413,
+          "expert_success_rate": 0.19480519480519481,
+          "num_groups": 77,
+          "oracle_success_rate": 0.36363636363636365,
+          "policy_expert_regret": 0.3055240137067127,
+          "policy_oracle_regret": 0.5390667356671645,
+          "policy_rollout_progress": 0.244317766750691,
+          "policy_rollout_success_rate": 0.1038961038961039,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.5283121121056537,
+          "expert_success_rate": 0.8617021276595744,
+          "num_groups": 94,
+          "oracle_success_rate": 0.9893617021276596,
+          "policy_expert_regret": 0.5605980893398853,
+          "policy_oracle_regret": 0.6775499712279502,
+          "policy_rollout_progress": 0.6735138585592838,
+          "policy_rollout_success_rate": 0.648936170212766,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.7289101645877265,
+          "expert_success_rate": 0.6767676767676768,
+          "num_groups": 99,
+          "oracle_success_rate": 0.8585858585858586,
+          "policy_expert_regret": 1.0163410561855393,
+          "policy_oracle_regret": 1.2083435260286235,
+          "policy_rollout_progress": 0.42377212402796505,
+          "policy_rollout_success_rate": 0.21212121212121213,
+          "restore_max_error": 3.948807716369629e-07
+        }
+      }
+    },
+    {
+      "seed": 1,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_1/retrieval_lattice_rollout.json",
+      "selection_mode": "retrieval_lattice",
+      "num_candidates": 16,
+      "num_groups": 575,
+      "policy_rollout_success_rate": 0.288695652173913,
+      "policy_rollout_progress": 0.500746654110414,
+      "oracle_success_rate": 0.8695652173913043,
+      "action_mse_to_best": 0.5489565526718355,
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.3817376020133693,
+          "expert_success_rate": 0.8584070796460177,
+          "num_groups": 113,
+          "oracle_success_rate": 0.9380530973451328,
+          "policy_expert_regret": 0.9911989344168554,
+          "policy_oracle_regret": 1.0726828230957015,
+          "policy_rollout_progress": 0.614330827266769,
+          "policy_rollout_success_rate": 0.25663716814159293,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.4809453334942784,
+          "expert_success_rate": 0.9402173913043478,
+          "num_groups": 184,
+          "oracle_success_rate": 0.9456521739130435,
+          "policy_expert_regret": 1.1634100335979145,
+          "policy_oracle_regret": 1.170946938565264,
+          "policy_rollout_progress": 0.5209976178264933,
+          "policy_rollout_success_rate": 0.2391304347826087,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.8191917944258373,
+          "expert_success_rate": 0.25,
+          "num_groups": 76,
+          "oracle_success_rate": 0.40789473684210525,
+          "policy_expert_regret": 0.3627372664119129,
+          "policy_oracle_regret": 0.6016990902357257,
+          "policy_rollout_progress": 0.2507582266507119,
+          "policy_rollout_success_rate": 0.11842105263157894,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.4591157559204746,
+          "expert_success_rate": 0.8198198198198198,
+          "num_groups": 111,
+          "oracle_success_rate": 1.0,
+          "policy_expert_regret": 0.5930133084724615,
+          "policy_oracle_regret": 0.7112691351422319,
+          "policy_rollout_progress": 0.6581002342271375,
+          "policy_rollout_success_rate": 0.6306306306306306,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.7780143089264959,
+          "expert_success_rate": 0.6923076923076923,
+          "num_groups": 91,
+          "oracle_success_rate": 0.8571428571428571,
+          "policy_expert_regret": 1.1573464636291777,
+          "policy_oracle_regret": 1.3298844198931705,
+          "policy_rollout_progress": 0.3356003266769451,
+          "policy_rollout_success_rate": 0.15384615384615385,
+          "restore_max_error": 3.948807716369629e-07
+        }
+      }
+    },
+    {
+      "seed": 2,
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_2/retrieval_lattice_rollout.json",
+      "selection_mode": "retrieval_lattice",
+      "num_candidates": 16,
+      "num_groups": 575,
+      "policy_rollout_success_rate": 0.29739130434782607,
+      "policy_rollout_progress": 0.5104680095143292,
+      "oracle_success_rate": 0.8765217391304347,
+      "action_mse_to_best": 0.5970965936059213,
+      "per_task": {
+        "LiftPegUpright-v1": {
+          "action_mse_to_best": 0.3823804637795547,
+          "expert_success_rate": 0.8229166666666666,
+          "num_groups": 96,
+          "oracle_success_rate": 0.9270833333333334,
+          "policy_expert_regret": 0.8790868987950186,
+          "policy_oracle_regret": 0.99962346519654,
+          "policy_rollout_progress": 0.6527350580630203,
+          "policy_rollout_success_rate": 0.28125,
+          "restore_max_error": 3.5762786865234375e-07
+        },
+        "PickCube-v1": {
+          "action_mse_to_best": 0.5080512181639371,
+          "expert_success_rate": 0.9444444444444444,
+          "num_groups": 198,
+          "oracle_success_rate": 0.9595959595959596,
+          "policy_expert_regret": 1.084108995668816,
+          "policy_oracle_regret": 1.0911924461975242,
+          "policy_rollout_progress": 0.5463541408229355,
+          "policy_rollout_success_rate": 0.30303030303030304,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "PullCube-v1": {
+          "action_mse_to_best": 0.8852970499338375,
+          "expert_success_rate": 0.24444444444444444,
+          "num_groups": 90,
+          "oracle_success_rate": 0.4666666666666667,
+          "policy_expert_regret": 0.3057925239722762,
+          "policy_oracle_regret": 0.5990210206247866,
+          "policy_rollout_progress": 0.307098902337667,
+          "policy_rollout_success_rate": 0.16666666666666666,
+          "restore_max_error": 4.0978193283081055e-07
+        },
+        "PushCube-v1": {
+          "action_mse_to_best": 0.45949358190640366,
+          "expert_success_rate": 0.8514851485148515,
+          "num_groups": 101,
+          "oracle_success_rate": 1.0,
+          "policy_expert_regret": 0.6463685169963553,
+          "policy_oracle_regret": 0.7728047379762819,
+          "policy_rollout_progress": 0.633135856083124,
+          "policy_rollout_success_rate": 0.594059405940594,
+          "restore_max_error": 4.76837158203125e-07
+        },
+        "StackCube-v1": {
+          "action_mse_to_best": 0.8882476593057315,
+          "expert_success_rate": 0.7666666666666667,
+          "num_groups": 90,
+          "oracle_success_rate": 0.9111111111111111,
+          "policy_expert_regret": 1.287395055923197,
+          "policy_oracle_regret": 1.4365394057499037,
+          "policy_rollout_progress": 0.3454755259884728,
+          "policy_rollout_success_rate": 0.1,
+          "restore_max_error": 4.76837158203125e-07
+        }
+      }
+    }
+  ]
+}

results/h16_retrieval_lattice_summary.md

@@ -0,0 +1,25 @@
+# h=16 Lattice-Selected Rollout
+
+Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs`
+Completed seeds: 3
+Baseline h=4 policy success: 29.67%
+Baseline h=16 policy success: 29.74%
+
+Mean success: 28.93% +/- 0.78%
+Gain vs h=16 policy: -0.81%
+Mean oracle success: 86.78%
+Mean progress: 50.70%
+
+| seed | success | progress | oracle | candidates | action MSE |
+|---:|---:|---:|---:|---:|---:|
+| 0 | 28.17% | 50.98% | 85.74% | 16 | 0.547 |
+| 1 | 28.87% | 50.07% | 86.96% | 16 | 0.549 |
+| 2 | 29.74% | 51.05% | 87.65% | 16 | 0.597 |
+
+Selected candidate types:
+- lattice_expert: 970
+- lattice_near_miss: 361
+- lattice_no_op: 172
+- lattice_random_negative: 133
+- lattice_wrong_direction: 25
+- lattice_wrong_gripper: 64