Auto-sync: 2026-06-27 10:13:44

results/h16_lattice_rollout_summary.json

@@ -1,225 +1,224 @@
 {
   "run_root": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs",
-  "out_name": "lattice_rollout.json",
+  "out_name": "lattice_no_expert_rollout.json",
   "num_completed": 3,
   "baseline_h4_policy_success": 0.2967,
   "baseline_h16_policy_success": 0.29739130434782607,
-  "mean_success": 0.6933333333333334,
-  "std_success": 0.035655708555783816,
-  "mean_progress": 0.8108624100247803,
+  "mean_success": 0.5698550724637681,
+  "std_success": 0.046188021535170105,
+  "mean_progress": 0.7501174603224449,
   "mean_oracle_success": 0.8678260869565217,
-  "mean_action_mse_to_best": 0.43754452051773696,
-  "gain_vs_h4": 0.39663333333333334,
-  "gain_vs_h16_policy": 0.3959420289855073,
+  "mean_action_mse_to_best": 0.4590102900127523,
+  "gain_vs_h4": 0.2731550724637681,
+  "gain_vs_h16_policy": 0.27246376811594203,
   "selected_candidate_type_counts": {
-    "lattice_expert": 977,
-    "lattice_near_miss": 348,
-    "lattice_no_op": 177,
-    "lattice_random_negative": 138,
-    "lattice_wrong_direction": 30,
-    "lattice_wrong_gripper": 55
+    "lattice_near_miss": 1263,
+    "lattice_no_op": 222,
+    "lattice_random_negative": 144,
+    "lattice_wrong_direction": 34,
+    "lattice_wrong_gripper": 62
   },
   "rows": [
     {
       "seed": 0,
-      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_0/lattice_rollout.json",
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_0/lattice_no_expert_rollout.json",
       "selection_mode": "lattice",
       "num_candidates": 16,
       "num_groups": 575,
-      "policy_rollout_success_rate": 0.6521739130434783,
-      "policy_rollout_progress": 0.792444871255885,
+      "policy_rollout_success_rate": 0.5165217391304348,
+      "policy_rollout_progress": 0.7308024795159049,
       "oracle_success_rate": 0.8573913043478261,
-      "action_mse_to_best": 0.414993303792207,
+      "action_mse_to_best": 0.4291479373977864,
       "per_task": {
         "LiftPegUpright-v1": {
-          "action_mse_to_best": 0.28245431445983055,
+          "action_mse_to_best": 0.2888141949958115,
           "expert_success_rate": 0.8865979381443299,
           "num_groups": 97,
           "oracle_success_rate": 0.9278350515463918,
-          "policy_expert_regret": 0.2610538084175169,
-          "policy_oracle_regret": 0.27425835227843415,
-          "policy_rollout_progress": 0.9071177396270418,
-          "policy_rollout_success_rate": 0.7216494845360825,
+          "policy_expert_regret": 0.45705366580142187,
+          "policy_oracle_regret": 0.47029784183526774,
+          "policy_rollout_progress": 0.8553990655645882,
+          "policy_rollout_success_rate": 0.5773195876288659,
           "restore_max_error": 4.76837158203125e-07
         },
         "PickCube-v1": {
-          "action_mse_to_best": 0.41056775828250325,
+          "action_mse_to_best": 0.4340394471700375,
           "expert_success_rate": 0.9375,
           "num_groups": 208,
           "oracle_success_rate": 0.9471153846153846,
-          "policy_expert_regret": 0.26980068733414203,
-          "policy_oracle_regret": 0.2764672613881815,
-          "policy_rollout_progress": 0.8699935431269785,
-          "policy_rollout_success_rate": 0.7788461538461539,
+          "policy_expert_regret": 0.6624748194769311,
+          "policy_oracle_regret": 0.6691413935309706,
+          "policy_rollout_progress": 0.7513381671697761,
+          "policy_rollout_success_rate": 0.5048076923076923,
           "restore_max_error": 4.76837158203125e-07
         },
         "PullCube-v1": {
-          "action_mse_to_best": 0.5239523940465667,
+          "action_mse_to_best": 0.5183976920587676,
           "expert_success_rate": 0.19480519480519481,
           "num_groups": 77,
           "oracle_success_rate": 0.36363636363636365,
-          "policy_expert_regret": 0.15346811585999154,
-          "policy_oracle_regret": 0.37490944806244464,
-          "policy_rollout_progress": 0.3196387503828321,
+          "policy_expert_regret": 0.1787171581058533,
+          "policy_oracle_regret": 0.37732240485100005,
+          "policy_rollout_progress": 0.3172257935942768,
           "policy_rollout_success_rate": 0.16883116883116883,
           "restore_max_error": 4.76837158203125e-07
         },
         "PushCube-v1": {
-          "action_mse_to_best": 0.38503194246143896,
+          "action_mse_to_best": 0.39197504613300826,
           "expert_success_rate": 0.8617021276595744,
           "num_groups": 94,
           "oracle_success_rate": 0.9893617021276596,
-          "policy_expert_regret": 0.28232911355952,
-          "policy_oracle_regret": 0.37923398661486646,
-          "policy_rollout_progress": 0.8140388716091501,
-          "policy_rollout_success_rate": 0.7872340425531915,
+          "policy_expert_regret": 0.2812512713543912,
+          "policy_oracle_regret": 0.3391457902307206,
+          "policy_rollout_progress": 0.8417052735990667,
+          "policy_rollout_success_rate": 0.8191489361702128,
           "restore_max_error": 4.76837158203125e-07
         },
         "StackCube-v1": {
-          "action_mse_to_best": 0.4978550557797361,
+          "action_mse_to_best": 0.5222485399266441,
           "expert_success_rate": 0.6767676767676768,
           "num_groups": 99,
           "oracle_success_rate": 0.8585858585858586,
-          "policy_expert_regret": 0.2738224623179195,
-          "policy_oracle_regret": 0.4027370100069528,
-          "policy_rollout_progress": 0.8643927833046576,
-          "policy_rollout_success_rate": 0.5656565656565656,
+          "policy_expert_regret": 0.5384464679342328,
+          "policy_oracle_regret": 0.574341721606977,
+          "policy_rollout_progress": 0.7819464218736899,
+          "policy_rollout_success_rate": 0.46464646464646464,
           "restore_max_error": 3.948807716369629e-07
         }
       }
     },
     {
       "seed": 1,
-      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_1/lattice_rollout.json",
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_1/lattice_no_expert_rollout.json",
       "selection_mode": "lattice",
       "num_candidates": 16,
       "num_groups": 575,
-      "policy_rollout_success_rate": 0.7147826086956521,
-      "policy_rollout_progress": 0.8205086112848442,
+      "policy_rollout_success_rate": 0.5965217391304348,
+      "policy_rollout_progress": 0.756582982868928,
       "oracle_success_rate": 0.8695652173913043,
-      "action_mse_to_best": 0.41892785357189893,
+      "action_mse_to_best": 0.44271026729079693,
       "per_task": {
         "LiftPegUpright-v1": {
-          "action_mse_to_best": 0.31486138409473396,
+          "action_mse_to_best": 0.3169037180301626,
           "expert_success_rate": 0.8584070796460177,
           "num_groups": 113,
           "oracle_success_rate": 0.9380530973451328,
-          "policy_expert_regret": 0.2672024345503444,
-          "policy_oracle_regret": 0.2922932648025783,
-          "policy_rollout_progress": 0.8941378234762006,
-          "policy_rollout_success_rate": 0.7345132743362832,
+          "policy_expert_regret": 0.39746501211044005,
+          "policy_oracle_regret": 0.43380374942733124,
+          "policy_rollout_progress": 0.858787138245802,
+          "policy_rollout_success_rate": 0.6283185840707964,
           "restore_max_error": 4.76837158203125e-07
         },
         "PickCube-v1": {
-          "action_mse_to_best": 0.44310908647173125,
+          "action_mse_to_best": 0.4862241648218554,
           "expert_success_rate": 0.9402173913043478,
           "num_groups": 184,
           "oracle_success_rate": 0.9456521739130435,
-          "policy_expert_regret": 0.17421010318819596,
-          "policy_oracle_regret": 0.1742341076631261,
-          "policy_rollout_progress": 0.9089716651436427,
-          "policy_rollout_success_rate": 0.8478260869565217,
+          "policy_expert_regret": 0.4408608768540232,
+          "policy_oracle_regret": 0.44088488132895337,
+          "policy_rollout_progress": 0.8249527744381971,
+          "policy_rollout_success_rate": 0.657608695652174,
           "restore_max_error": 4.76837158203125e-07
         },
         "PullCube-v1": {
-          "action_mse_to_best": 0.5217870954473159,
+          "action_mse_to_best": 0.5220103718977618,
           "expert_success_rate": 0.25,
           "num_groups": 76,
           "oracle_success_rate": 0.40789473684210525,
-          "policy_expert_regret": 0.12901650541952173,
-          "policy_oracle_regret": 0.3323121542037514,
-          "policy_rollout_progress": 0.379051765659824,
-          "policy_rollout_success_rate": 0.23684210526315788,
+          "policy_expert_regret": 0.2234261535524734,
+          "policy_oracle_regret": 0.44941299718706623,
+          "policy_rollout_progress": 0.32774039636071967,
+          "policy_rollout_success_rate": 0.17105263157894737,
           "restore_max_error": 4.76837158203125e-07
         },
         "PushCube-v1": {
-          "action_mse_to_best": 0.34412760804365344,
+          "action_mse_to_best": 0.35074781935225735,
           "expert_success_rate": 0.8198198198198198,
           "num_groups": 111,
           "oracle_success_rate": 1.0,
-          "policy_expert_regret": 0.1992848778630162,
-          "policy_oracle_regret": 0.3170433482071301,
-          "policy_rollout_progress": 0.8541278229640411,
-          "policy_rollout_success_rate": 0.8288288288288288,
+          "policy_expert_regret": 0.24994636481409674,
+          "policy_oracle_regret": 0.3667086133280316,
+          "policy_rollout_progress": 0.8314895848701666,
+          "policy_rollout_success_rate": 0.8018018018018018,
           "restore_max_error": 4.76837158203125e-07
         },
         "StackCube-v1": {
-          "action_mse_to_best": 0.5045947664120531,
+          "action_mse_to_best": 0.5568927584093187,
           "expert_success_rate": 0.6923076923076923,
           "num_groups": 91,
           "oracle_success_rate": 0.8571428571428571,
-          "policy_expert_regret": 0.1715098601448667,
-          "policy_oracle_regret": 0.2727490429367338,
-          "policy_rollout_progress": 0.8778897638504322,
-          "policy_rollout_success_rate": 0.6813186813186813,
+          "policy_expert_regret": 0.47163032846791403,
+          "policy_oracle_regret": 0.5352666910205569,
+          "policy_rollout_progress": 0.7582125712882032,
+          "policy_rollout_success_rate": 0.5384615384615384,
           "restore_max_error": 3.948807716369629e-07
         }
       }
     },
     {
       "seed": 2,
-      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_2/lattice_rollout.json",
+      "path": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_2/lattice_no_expert_rollout.json",
       "selection_mode": "lattice",
       "num_candidates": 16,
       "num_groups": 575,
-      "policy_rollout_success_rate": 0.7130434782608696,
-      "policy_rollout_progress": 0.8196337475336116,
+      "policy_rollout_success_rate": 0.5965217391304348,
+      "policy_rollout_progress": 0.7629669185825017,
       "oracle_success_rate": 0.8765217391304347,
-      "action_mse_to_best": 0.47871240418910493,
+      "action_mse_to_best": 0.5051726653496735,
       "per_task": {
         "LiftPegUpright-v1": {
-          "action_mse_to_best": 0.33342877044075675,
+          "action_mse_to_best": 0.3387383111627666,
           "expert_success_rate": 0.8229166666666666,
           "num_groups": 96,
           "oracle_success_rate": 0.9270833333333334,
-          "policy_expert_regret": 0.22194465808570385,
-          "policy_oracle_regret": 0.28883908068140346,
-          "policy_rollout_progress": 0.9184654237081608,
-          "policy_rollout_success_rate": 0.7395833333333334,
+          "policy_expert_regret": 0.349092789615194,
+          "policy_oracle_regret": 0.38970588706433773,
+          "policy_rollout_progress": 0.8847468371192614,
+          "policy_rollout_success_rate": 0.6458333333333334,
           "restore_max_error": 3.5762786865234375e-07
         },
         "PickCube-v1": {
-          "action_mse_to_best": 0.4627937324570887,
+          "action_mse_to_best": 0.5211711752068515,
           "expert_success_rate": 0.9444444444444444,
           "num_groups": 198,
           "oracle_success_rate": 0.9595959595959596,
-          "policy_expert_regret": 0.10305700688199564,
-          "policy_oracle_regret": 0.11014045741070401,
-          "policy_rollout_progress": 0.9385366825518584,
-          "policy_rollout_success_rate": 0.898989898989899,
+          "policy_expert_regret": 0.38791667779136185,
+          "policy_oracle_regret": 0.38796894105546403,
+          "policy_rollout_progress": 0.8505876985086938,
+          "policy_rollout_success_rate": 0.702020202020202,
           "restore_max_error": 4.76837158203125e-07
         },
         "PullCube-v1": {
-          "action_mse_to_best": 0.6827966811549332,
+          "action_mse_to_best": 0.682747141416702,
           "expert_success_rate": 0.24444444444444444,
           "num_groups": 90,
           "oracle_success_rate": 0.4666666666666667,
-          "policy_expert_regret": 0.15796189763479762,
-          "policy_oracle_regret": 0.40946051337652734,
-          "policy_rollout_progress": 0.39318432178762225,
-          "policy_rollout_success_rate": 0.26666666666666666,
+          "policy_expert_regret": 0.2159867504404651,
+          "policy_oracle_regret": 0.4278881938921081,
+          "policy_rollout_progress": 0.38586775238315263,
+          "policy_rollout_success_rate": 0.25555555555555554,
           "restore_max_error": 4.0978193283081055e-07
         },
         "PushCube-v1": {
-          "action_mse_to_best": 0.34167051138264115,
+          "action_mse_to_best": 0.3426748497024326,
           "expert_success_rate": 0.8514851485148515,
           "num_groups": 101,
           "oracle_success_rate": 1.0,
-          "policy_expert_regret": 0.24242213811024582,
-          "policy_oracle_regret": 0.29742342971339086,
-          "policy_rollout_progress": 0.8609924118707676,
-          "policy_rollout_success_rate": 0.8415841584158416,
+          "policy_expert_regret": 0.33127955694009764,
+          "policy_oracle_regret": 0.40476624254543003,
+          "policy_rollout_progress": 0.8130555396327878,
+          "policy_rollout_success_rate": 0.7821782178217822,
           "restore_max_error": 4.76837158203125e-07
         },
         "StackCube-v1": {
-          "action_mse_to_best": 0.6184098718480932,
+          "action_mse_to_best": 0.6522901051781244,
           "expert_success_rate": 0.7666666666666667,
           "num_groups": 90,
           "oracle_success_rate": 0.9111111111111111,
-          "policy_expert_regret": 0.3941795117325253,
-          "policy_oracle_regret": 0.4843028916252984,
-          "policy_rollout_progress": 0.8326626494526863,
-          "policy_rollout_success_rate": 0.5777777777777777,
+          "policy_expert_regret": 0.6501921941836675,
+          "policy_oracle_regret": 0.6890770451890098,
+          "policy_rollout_progress": 0.7611912255485852,
+          "policy_rollout_success_rate": 0.4444444444444444,
           "restore_max_error": 4.76837158203125e-07
         }
       }

results/h16_lattice_rollout_summary.md

@@ -5,21 +5,20 @@ Completed seeds: 3
 Baseline h=4 policy success: 29.67%
 Baseline h=16 policy success: 29.74%
 
-Mean success: 69.33% +/- 3.57%
-Gain vs h=16 policy: +39.59%
+Mean success: 56.99% +/- 4.62%
+Gain vs h=16 policy: +27.25%
 Mean oracle success: 86.78%
-Mean progress: 81.09%
+Mean progress: 75.01%
 
 | seed | success | progress | oracle | candidates | action MSE |
 |---:|---:|---:|---:|---:|---:|
-| 0 | 65.22% | 79.24% | 85.74% | 16 | 0.415 |
-| 1 | 71.48% | 82.05% | 86.96% | 16 | 0.419 |
-| 2 | 71.30% | 81.96% | 87.65% | 16 | 0.479 |
+| 0 | 51.65% | 73.08% | 85.74% | 16 | 0.429 |
+| 1 | 59.65% | 75.66% | 86.96% | 16 | 0.443 |
+| 2 | 59.65% | 76.30% | 87.65% | 16 | 0.505 |
 
 Selected candidate types:
-- lattice_expert: 977
-- lattice_near_miss: 348
-- lattice_no_op: 177
-- lattice_random_negative: 138
-- lattice_wrong_direction: 30
-- lattice_wrong_gripper: 55
+- lattice_near_miss: 1263
+- lattice_no_op: 222
+- lattice_random_negative: 144
+- lattice_wrong_direction: 34
+- lattice_wrong_gripper: 62