auto-sync 2026-07-02T13:37:00Z workspace (part 2)

workspace/logs/enhanced_train_14687215_1.out

@@ -0,0 +1,34 @@
+= = = = = = = = = = = = = = = = = =
+DoVLA-Attention-Enhanced: SOTA Training
+= = = = = = = = = = = = = = = = = =
+
+Architecture Components:
+  1. Hierarchical Attention (local + global)
+  2. Graph Neural Network (action relationships)
+  3. Contrastive Learning (better embeddings)
+  4. Task-Adaptive Layers (multi-task)
+
+Dataset: 3,500 groups (fair comparison)
+Seed: 1
+
+Expected: 44-47% success (vs 38.43% baseline)
+Improvement: +5.5-8.5%
+
+======================================================================
+Enhanced DoVLA-Attention Training (CVPR)
+======================================================================
+Dataset: /scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection
+Device: cuda
+Architecture: Hierarchical + Graph + Contrastive + Task-Adaptive
+Hidden: 256, Heads: 4, Layers: 3
+Seed: 1
+
+Loading dataset...
+Total: 3500, Train: 2800, Val: 700
+
+Observation dim: 70, Action dim: 32
+
+Model parameters: 4,374,401
+
+Starting training...
+

workspace/logs/enhanced_train_14687215_2.err

@@ -0,0 +1,20 @@
+Traceback (most recent call last):
+  File "/lustre09/project/6037638/knguy52/vla/scripts/train_dovla_enhanced.py", line 407, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/train_dovla_enhanced.py", line 353, in main
+    train_metrics = train_epoch(model, train_loader, optimizer, device, args.contrastive_weight)
+                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/train_dovla_enhanced.py", line 170, in train_epoch
+    scores, contrastive_loss = model(obs, actions, task_ids, rewards)
+                               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/dovla_cil/models/dovla_attention_enhanced.py", line 452, in forward
+    cos_sim = F.cosine_similarity(h_i, h_j, dim=-1, keepdim=True)
+              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+TypeError: cosine_similarity() got an unexpected keyword argument 'keepdim'

workspace/logs/enhanced_train_14687215_2.out

@@ -0,0 +1,34 @@
+= = = = = = = = = = = = = = = = = =
+DoVLA-Attention-Enhanced: SOTA Training
+= = = = = = = = = = = = = = = = = =
+
+Architecture Components:
+  1. Hierarchical Attention (local + global)
+  2. Graph Neural Network (action relationships)
+  3. Contrastive Learning (better embeddings)
+  4. Task-Adaptive Layers (multi-task)
+
+Dataset: 3,500 groups (fair comparison)
+Seed: 2
+
+Expected: 44-47% success (vs 38.43% baseline)
+Improvement: +5.5-8.5%
+
+======================================================================
+Enhanced DoVLA-Attention Training (CVPR)
+======================================================================
+Dataset: /scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection
+Device: cuda
+Architecture: Hierarchical + Graph + Contrastive + Task-Adaptive
+Hidden: 256, Heads: 4, Layers: 3
+Seed: 2
+
+Loading dataset...
+Total: 3500, Train: 2800, Val: 700
+
+Observation dim: 70, Action dim: 32
+
+Model parameters: 4,374,401
+
+Starting training...
+

workspace/logs/enhanced_train_14687360_0.out

@@ -0,0 +1,90 @@
+= = = = = = = = = = = = = = = = = =
+DoVLA-Attention-Enhanced: SOTA Training
+= = = = = = = = = = = = = = = = = =
+
+Architecture Components:
+  1. Hierarchical Attention (local + global)
+  2. Graph Neural Network (action relationships)
+  3. Contrastive Learning (better embeddings)
+  4. Task-Adaptive Layers (multi-task)
+
+Dataset: 3,500 groups (fair comparison)
+Seed: 0
+
+Expected: 44-47% success (vs 38.43% baseline)
+Improvement: +5.5-8.5%
+
+======================================================================
+Enhanced DoVLA-Attention Training (CVPR)
+======================================================================
+Dataset: /scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection
+Device: cuda
+Architecture: Hierarchical + Graph + Contrastive + Task-Adaptive
+Hidden: 256, Heads: 4, Layers: 3
+Seed: 0
+
+Loading dataset...
+Total: 3500, Train: 2800, Val: 700
+
+Observation dim: 70, Action dim: 32
+
+Model parameters: 4,374,401
+
+Starting training...
+
+Epoch   1/50: rank_loss=0.6980, contr_loss=1.2153, val_acc=0.5000
+Epoch   2/50: rank_loss=0.6943, contr_loss=0.9573, val_acc=0.5000
+Epoch   3/50: rank_loss=0.6940, contr_loss=0.9303, val_acc=0.5000
+Epoch   4/50: rank_loss=0.6937, contr_loss=0.8816, val_acc=0.5000
+Epoch   5/50: rank_loss=0.6935, contr_loss=0.8862, val_acc=0.5000
+Epoch   6/50: rank_loss=0.6935, contr_loss=0.8504, val_acc=0.5000
+Epoch   7/50: rank_loss=0.6934, contr_loss=0.7998, val_acc=0.5000
+Epoch   8/50: rank_loss=0.6934, contr_loss=0.8008, val_acc=0.5000
+Epoch   9/50: rank_loss=0.6933, contr_loss=0.8075, val_acc=0.5000
+Epoch  10/50: rank_loss=0.6933, contr_loss=0.8081, val_acc=0.5000
+Epoch  11/50: rank_loss=0.6933, contr_loss=0.8006, val_acc=0.5000
+Epoch  12/50: rank_loss=0.6933, contr_loss=0.8085, val_acc=0.5000
+Epoch  13/50: rank_loss=0.6932, contr_loss=0.7825, val_acc=0.5000
+Epoch  14/50: rank_loss=0.6932, contr_loss=0.8122, val_acc=0.5000
+Epoch  15/50: rank_loss=0.6932, contr_loss=0.8361, val_acc=0.5000
+Epoch  16/50: rank_loss=0.6932, contr_loss=0.8358, val_acc=0.5000
+Epoch  17/50: rank_loss=0.6932, contr_loss=0.8506, val_acc=0.5000
+Epoch  18/50: rank_loss=0.6932, contr_loss=0.8049, val_acc=0.5000
+Epoch  19/50: rank_loss=0.6932, contr_loss=0.7821, val_acc=0.5000
+Epoch  20/50: rank_loss=0.6932, contr_loss=0.8272, val_acc=0.5000
+Epoch  21/50: rank_loss=0.6932, contr_loss=0.7854, val_acc=0.5000
+Epoch  22/50: rank_loss=0.6932, contr_loss=0.7700, val_acc=0.5000
+Epoch  23/50: rank_loss=0.6932, contr_loss=0.7742, val_acc=0.5000
+Epoch  24/50: rank_loss=0.6932, contr_loss=0.7773, val_acc=0.5000
+Epoch  25/50: rank_loss=0.6932, contr_loss=0.7843, val_acc=0.5000
+Epoch  26/50: rank_loss=0.6932, contr_loss=0.7495, val_acc=0.5000
+Epoch  27/50: rank_loss=0.6932, contr_loss=0.7527, val_acc=0.5000
+Epoch  28/50: rank_loss=0.6932, contr_loss=0.7521, val_acc=0.5000
+Epoch  29/50: rank_loss=0.6932, contr_loss=0.7476, val_acc=0.5000
+Epoch  30/50: rank_loss=0.6932, contr_loss=0.7416, val_acc=0.5000
+Epoch  31/50: rank_loss=0.6932, contr_loss=0.7506, val_acc=0.5000
+Epoch  32/50: rank_loss=0.6932, contr_loss=0.7348, val_acc=0.5000
+Epoch  33/50: rank_loss=0.6932, contr_loss=0.7364, val_acc=0.5000
+Epoch  34/50: rank_loss=0.6932, contr_loss=0.7411, val_acc=0.5000
+Epoch  35/50: rank_loss=0.6932, contr_loss=0.7443, val_acc=0.5000
+Epoch  36/50: rank_loss=0.6932, contr_loss=0.7310, val_acc=0.5000
+Epoch  37/50: rank_loss=0.6931, contr_loss=0.7176, val_acc=0.5000
+Epoch  38/50: rank_loss=0.6932, contr_loss=0.7172, val_acc=0.5000
+Epoch  39/50: rank_loss=0.6932, contr_loss=0.7241, val_acc=0.5000
+Epoch  40/50: rank_loss=0.6932, contr_loss=0.7246, val_acc=0.5000
+Epoch  41/50: rank_loss=0.6932, contr_loss=0.7208, val_acc=0.5000
+Epoch  42/50: rank_loss=0.6932, contr_loss=0.7213, val_acc=0.5000
+Epoch  43/50: rank_loss=0.6932, contr_loss=0.7230, val_acc=0.5000
+Epoch  44/50: rank_loss=0.6932, contr_loss=0.7242, val_acc=0.5000
+Epoch  45/50: rank_loss=0.6932, contr_loss=0.7251, val_acc=0.5000
+Epoch  46/50: rank_loss=0.6932, contr_loss=0.7129, val_acc=0.5000
+Epoch  47/50: rank_loss=0.6932, contr_loss=0.7098, val_acc=0.5000
+Epoch  48/50: rank_loss=0.6932, contr_loss=0.7206, val_acc=0.5000
+Epoch  49/50: rank_loss=0.6932, contr_loss=0.7259, val_acc=0.5000
+Epoch  50/50: rank_loss=0.6932, contr_loss=0.7190, val_acc=0.5000
+
+✅ Training complete! Best val accuracy: 0.5000
+
+✅ Enhanced training complete (seed 0)
+
+Next: Evaluate and compare with baseline

workspace/logs/enhanced_train_14687360_1.out

@@ -0,0 +1,90 @@
+= = = = = = = = = = = = = = = = = =
+DoVLA-Attention-Enhanced: SOTA Training
+= = = = = = = = = = = = = = = = = =
+
+Architecture Components:
+  1. Hierarchical Attention (local + global)
+  2. Graph Neural Network (action relationships)
+  3. Contrastive Learning (better embeddings)
+  4. Task-Adaptive Layers (multi-task)
+
+Dataset: 3,500 groups (fair comparison)
+Seed: 1
+
+Expected: 44-47% success (vs 38.43% baseline)
+Improvement: +5.5-8.5%
+
+======================================================================
+Enhanced DoVLA-Attention Training (CVPR)
+======================================================================
+Dataset: /scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection
+Device: cuda
+Architecture: Hierarchical + Graph + Contrastive + Task-Adaptive
+Hidden: 256, Heads: 4, Layers: 3
+Seed: 1
+
+Loading dataset...
+Total: 3500, Train: 2800, Val: 700
+
+Observation dim: 70, Action dim: 32
+
+Model parameters: 4,374,401
+
+Starting training...
+
+Epoch   1/50: rank_loss=0.7001, contr_loss=1.1553, val_acc=0.5000
+Epoch   2/50: rank_loss=0.6943, contr_loss=0.9068, val_acc=0.5000
+Epoch   3/50: rank_loss=0.6938, contr_loss=0.8833, val_acc=0.5000
+Epoch   4/50: rank_loss=0.6936, contr_loss=0.8913, val_acc=0.5000
+Epoch   5/50: rank_loss=0.6935, contr_loss=0.8622, val_acc=0.5000
+Epoch   6/50: rank_loss=0.6934, contr_loss=0.8331, val_acc=0.5000
+Epoch   7/50: rank_loss=0.6934, contr_loss=0.8409, val_acc=0.5000
+Epoch   8/50: rank_loss=0.6933, contr_loss=0.8098, val_acc=0.5000
+Epoch   9/50: rank_loss=0.6933, contr_loss=0.7961, val_acc=0.5000
+Epoch  10/50: rank_loss=0.6933, contr_loss=0.8179, val_acc=0.5000
+Epoch  11/50: rank_loss=0.6932, contr_loss=0.8008, val_acc=0.5000
+Epoch  12/50: rank_loss=0.6932, contr_loss=0.7937, val_acc=0.5000
+Epoch  13/50: rank_loss=0.6932, contr_loss=0.8018, val_acc=0.5000
+Epoch  14/50: rank_loss=0.6932, contr_loss=0.8028, val_acc=0.5000
+Epoch  15/50: rank_loss=0.6932, contr_loss=0.7858, val_acc=0.5000
+Epoch  16/50: rank_loss=0.6932, contr_loss=0.7927, val_acc=0.5000
+Epoch  17/50: rank_loss=0.6932, contr_loss=0.7666, val_acc=0.5000
+Epoch  18/50: rank_loss=0.6932, contr_loss=0.7621, val_acc=0.5000
+Epoch  19/50: rank_loss=0.6932, contr_loss=0.7803, val_acc=0.5000
+Epoch  20/50: rank_loss=0.6932, contr_loss=0.7816, val_acc=0.5000
+Epoch  21/50: rank_loss=0.6932, contr_loss=0.7781, val_acc=0.5000
+Epoch  22/50: rank_loss=0.6932, contr_loss=0.7827, val_acc=0.5000
+Epoch  23/50: rank_loss=0.6932, contr_loss=0.7614, val_acc=0.5000
+Epoch  24/50: rank_loss=0.6932, contr_loss=0.7608, val_acc=0.5000
+Epoch  25/50: rank_loss=0.6932, contr_loss=0.7596, val_acc=0.5000
+Epoch  26/50: rank_loss=0.6932, contr_loss=0.7664, val_acc=0.5000
+Epoch  27/50: rank_loss=0.6932, contr_loss=0.7411, val_acc=0.5000
+Epoch  28/50: rank_loss=0.6932, contr_loss=0.7323, val_acc=0.5000
+Epoch  29/50: rank_loss=0.6932, contr_loss=0.7416, val_acc=0.5000
+Epoch  30/50: rank_loss=0.6932, contr_loss=0.7378, val_acc=0.5000
+Epoch  31/50: rank_loss=0.6932, contr_loss=0.7480, val_acc=0.5000
+Epoch  32/50: rank_loss=0.6932, contr_loss=0.7435, val_acc=0.5000
+Epoch  33/50: rank_loss=0.6932, contr_loss=0.7372, val_acc=0.5000
+Epoch  34/50: rank_loss=0.6932, contr_loss=0.7290, val_acc=0.5000
+Epoch  35/50: rank_loss=0.6932, contr_loss=0.7248, val_acc=0.5000
+Epoch  36/50: rank_loss=0.6932, contr_loss=0.7199, val_acc=0.5000
+Epoch  37/50: rank_loss=0.6932, contr_loss=0.7109, val_acc=0.5000
+Epoch  38/50: rank_loss=0.6932, contr_loss=0.7272, val_acc=0.5000
+Epoch  39/50: rank_loss=0.6932, contr_loss=0.7206, val_acc=0.5000
+Epoch  40/50: rank_loss=0.6931, contr_loss=0.7030, val_acc=0.5000
+Epoch  41/50: rank_loss=0.6932, contr_loss=0.7069, val_acc=0.5000
+Epoch  42/50: rank_loss=0.6932, contr_loss=0.7082, val_acc=0.5000
+Epoch  43/50: rank_loss=0.6932, contr_loss=0.7044, val_acc=0.5000
+Epoch  44/50: rank_loss=0.6932, contr_loss=0.6986, val_acc=0.5000
+Epoch  45/50: rank_loss=0.6932, contr_loss=0.7028, val_acc=0.5000
+Epoch  46/50: rank_loss=0.6932, contr_loss=0.6937, val_acc=0.5000
+Epoch  47/50: rank_loss=0.6932, contr_loss=0.7020, val_acc=0.5000
+Epoch  48/50: rank_loss=0.6932, contr_loss=0.7054, val_acc=0.5000
+Epoch  49/50: rank_loss=0.6932, contr_loss=0.7040, val_acc=0.5000
+Epoch  50/50: rank_loss=0.6931, contr_loss=0.6985, val_acc=0.5000
+
+✅ Training complete! Best val accuracy: 0.5000
+
+✅ Enhanced training complete (seed 1)
+
+Next: Evaluate and compare with baseline

workspace/logs/enhanced_train_14687360_2.out

@@ -0,0 +1,90 @@
+= = = = = = = = = = = = = = = = = =
+DoVLA-Attention-Enhanced: SOTA Training
+= = = = = = = = = = = = = = = = = =
+
+Architecture Components:
+  1. Hierarchical Attention (local + global)
+  2. Graph Neural Network (action relationships)
+  3. Contrastive Learning (better embeddings)
+  4. Task-Adaptive Layers (multi-task)
+
+Dataset: 3,500 groups (fair comparison)
+Seed: 2
+
+Expected: 44-47% success (vs 38.43% baseline)
+Improvement: +5.5-8.5%
+
+======================================================================
+Enhanced DoVLA-Attention Training (CVPR)
+======================================================================
+Dataset: /scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection
+Device: cuda
+Architecture: Hierarchical + Graph + Contrastive + Task-Adaptive
+Hidden: 256, Heads: 4, Layers: 3
+Seed: 2
+
+Loading dataset...
+Total: 3500, Train: 2800, Val: 700
+
+Observation dim: 70, Action dim: 32
+
+Model parameters: 4,374,401
+
+Starting training...
+
+Epoch   1/50: rank_loss=0.6992, contr_loss=1.1632, val_acc=0.5000
+Epoch   2/50: rank_loss=0.6944, contr_loss=0.9737, val_acc=0.5000
+Epoch   3/50: rank_loss=0.6939, contr_loss=0.8806, val_acc=0.5000
+Epoch   4/50: rank_loss=0.6936, contr_loss=0.8334, val_acc=0.5000
+Epoch   5/50: rank_loss=0.6935, contr_loss=0.8105, val_acc=0.5000
+Epoch   6/50: rank_loss=0.6934, contr_loss=0.7987, val_acc=0.5000
+Epoch   7/50: rank_loss=0.6933, contr_loss=0.8100, val_acc=0.5000
+Epoch   8/50: rank_loss=0.6933, contr_loss=0.8002, val_acc=0.5000
+Epoch   9/50: rank_loss=0.6933, contr_loss=0.7978, val_acc=0.5000
+Epoch  10/50: rank_loss=0.6933, contr_loss=0.7791, val_acc=0.5000
+Epoch  11/50: rank_loss=0.6933, contr_loss=0.7786, val_acc=0.5000
+Epoch  12/50: rank_loss=0.6932, contr_loss=0.7700, val_acc=0.5000
+Epoch  13/50: rank_loss=0.6932, contr_loss=0.7748, val_acc=0.5000
+Epoch  14/50: rank_loss=0.6932, contr_loss=0.7871, val_acc=0.5000
+Epoch  15/50: rank_loss=0.6932, contr_loss=0.7829, val_acc=0.5000
+Epoch  16/50: rank_loss=0.6932, contr_loss=0.7693, val_acc=0.5000
+Epoch  17/50: rank_loss=0.6932, contr_loss=0.7735, val_acc=0.5000
+Epoch  18/50: rank_loss=0.6932, contr_loss=0.7906, val_acc=0.5000
+Epoch  19/50: rank_loss=0.6932, contr_loss=0.7690, val_acc=0.5000
+Epoch  20/50: rank_loss=0.6932, contr_loss=0.7729, val_acc=0.5000
+Epoch  21/50: rank_loss=0.6932, contr_loss=0.7561, val_acc=0.5000
+Epoch  22/50: rank_loss=0.6932, contr_loss=0.7520, val_acc=0.5000
+Epoch  23/50: rank_loss=0.6932, contr_loss=0.7322, val_acc=0.5000
+Epoch  24/50: rank_loss=0.6932, contr_loss=0.7429, val_acc=0.5000
+Epoch  25/50: rank_loss=0.6932, contr_loss=0.7537, val_acc=0.5000
+Epoch  26/50: rank_loss=0.6932, contr_loss=0.7484, val_acc=0.5000
+Epoch  27/50: rank_loss=0.6932, contr_loss=0.7296, val_acc=0.5000
+Epoch  28/50: rank_loss=0.6932, contr_loss=0.7449, val_acc=0.5000
+Epoch  29/50: rank_loss=0.6932, contr_loss=0.7376, val_acc=0.5000
+Epoch  30/50: rank_loss=0.6932, contr_loss=0.7225, val_acc=0.5000
+Epoch  31/50: rank_loss=0.6932, contr_loss=0.7342, val_acc=0.5000
+Epoch  32/50: rank_loss=0.6932, contr_loss=0.7216, val_acc=0.5000
+Epoch  33/50: rank_loss=0.6932, contr_loss=0.7252, val_acc=0.5000
+Epoch  34/50: rank_loss=0.6932, contr_loss=0.7211, val_acc=0.5000
+Epoch  35/50: rank_loss=0.6932, contr_loss=0.7251, val_acc=0.5000
+Epoch  36/50: rank_loss=0.6932, contr_loss=0.7277, val_acc=0.5000
+Epoch  37/50: rank_loss=0.6932, contr_loss=0.7172, val_acc=0.5000
+Epoch  38/50: rank_loss=0.6932, contr_loss=0.7202, val_acc=0.5000
+Epoch  39/50: rank_loss=0.6932, contr_loss=0.7221, val_acc=0.5000
+Epoch  40/50: rank_loss=0.6932, contr_loss=0.7178, val_acc=0.5000
+Epoch  41/50: rank_loss=0.6932, contr_loss=0.7090, val_acc=0.5000
+Epoch  42/50: rank_loss=0.6931, contr_loss=0.7153, val_acc=0.5000
+Epoch  43/50: rank_loss=0.6932, contr_loss=0.7104, val_acc=0.5000
+Epoch  44/50: rank_loss=0.6932, contr_loss=0.7146, val_acc=0.5000
+Epoch  45/50: rank_loss=0.6932, contr_loss=0.7022, val_acc=0.5000
+Epoch  46/50: rank_loss=0.6932, contr_loss=0.7082, val_acc=0.5000
+Epoch  47/50: rank_loss=0.6932, contr_loss=0.7097, val_acc=0.5000
+Epoch  48/50: rank_loss=0.6932, contr_loss=0.7035, val_acc=0.5000
+Epoch  49/50: rank_loss=0.6931, contr_loss=0.7078, val_acc=0.5000
+Epoch  50/50: rank_loss=0.6931, contr_loss=0.7112, val_acc=0.5000
+
+✅ Training complete! Best val accuracy: 0.5000
+
+✅ Enhanced training complete (seed 2)
+
+Next: Evaluate and compare with baseline

workspace/logs/eval_a1_revised_14664453.out

@@ -0,0 +1,484 @@
+=== Evaluating Phase A1-Revised Enhanced Models ===
+
+Evaluating seed 0...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a1_revised_enhanced/seed_0/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8507781576673159,
+  "top1_action_selection": 0.6114285714285714,
+  "selected_success_rate": 0.37857142857142856,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9746324906196052,
+  "potential_edge_mae": 0.29218482069354706,
+  "effect_prediction_mae": 0.025904718118875455,
+  "selection_regret": 0.07466521533977773,
+  "selected_candidate_type_counts": {
+    "expert": 1862,
+    "near_miss": 611,
+    "no_op": 419,
+    "random_negative": 30,
+    "wrong_direction": 205,
+    "wrong_gripper": 373
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8372760104370064,
+      "top1_action_selection": 0.572,
+      "selected_success_rate": 0.404,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9673779491057297,
+      "potential_edge_mae": 0.3259476305709723,
+      "effect_prediction_mae": 0.02046124974363364,
+      "selection_regret": 0.1150644493997097,
+      "selected_candidate_type_counts": {
+        "expert": 351,
+        "near_miss": 81,
+        "no_op": 3,
+        "random_negative": 2,
+        "wrong_direction": 20,
+        "wrong_gripper": 43
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8198839574510653,
+      "top1_action_selection": 0.682,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9737146401786494,
+      "potential_edge_mae": 0.17428899767333667,
+      "effect_prediction_mae": 0.04585162171532977,
+      "selection_regret": 0.03477694494090974,
+      "selected_candidate_type_counts": {
+        "expert": 284,
+        "near_miss": 14,
+        "wrong_direction": 7,
+        "wrong_gripper": 195
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8755635632280231,
+      "top1_action_selection": 0.524,
+      "selected_success_rate": 0.318,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.975776432575271,
+      "potential_edge_mae": 0.2910050306161701,
+      "effect_prediction_mae": 0.020499905351573436,
+      "selection_regret": 0.09195959524065256,
+      "selected_candidate_type_counts": {
+        "expert": 400,
+        "near_miss": 163,
+        "no_op": 408,
+        "wrong_direction": 27,
+        "wrong_gripper": 2
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8435432413335332,
+      "top1_action_selection": 0.698,
+      "selected_success_rate": 0.608,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9813948972377147,
+      "potential_edge_mae": 0.27489662176823304,
+      "effect_prediction_mae": 0.019396800087932185,
+      "selection_regret": 0.045078758046030995,
+      "selected_candidate_type_counts": {
+        "expert": 259,
+        "near_miss": 135,
+        "random_negative": 12,
+        "wrong_direction": 75,
+        "wrong_gripper": 19
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8616394420824942,
+      "top1_action_selection": 0.718,
+      "selected_success_rate": 0.652,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9781340173951859,
+      "potential_edge_mae": 0.38579861025676476,
+      "effect_prediction_mae": 0.022150712490473203,
+      "selection_regret": 0.05170745313167572,
+      "selected_candidate_type_counts": {
+        "expert": 264,
+        "near_miss": 83,
+        "no_op": 3,
+        "random_negative": 16,
+        "wrong_direction": 44,
+        "wrong_gripper": 90
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8412163868729899,
+      "top1_action_selection": 0.562,
+      "selected_success_rate": 0.34,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9702530652693935,
+      "potential_edge_mae": 0.30978307795222954,
+      "effect_prediction_mae": 0.032472832091612794,
+      "selection_regret": 0.09210971137881278,
+      "selected_candidate_type_counts": {
+        "expert": 304,
+        "near_miss": 135,
+        "no_op": 5,
+        "wrong_direction": 32,
+        "wrong_gripper": 24
+      }
+    }
+  }
+}
+✅ Seed 0 complete
+   Success: 0.3786 | Top1: 0.6114 | Rank: 0.8508
+
+Evaluating seed 1...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a1_revised_enhanced/seed_1/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 1,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8470038977170514,
+  "top1_action_selection": 0.596,
+  "selected_success_rate": 0.37657142857142856,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9720251036674814,
+  "potential_edge_mae": 0.2730792666363643,
+  "effect_prediction_mae": 0.02844222002923243,
+  "selection_regret": 0.07795452343299986,
+  "selected_candidate_type_counts": {
+    "expert": 1979,
+    "near_miss": 616,
+    "no_op": 377,
+    "random_negative": 101,
+    "wrong_direction": 139,
+    "wrong_gripper": 288
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8315736724784434,
+      "top1_action_selection": 0.554,
+      "selected_success_rate": 0.4,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9650526053407107,
+      "potential_edge_mae": 0.28923494204159383,
+      "effect_prediction_mae": 0.02350660307392276,
+      "selection_regret": 0.11971647167205811,
+      "selected_candidate_type_counts": {
+        "expert": 288,
+        "near_miss": 151,
+        "random_negative": 4,
+        "wrong_direction": 13,
+        "wrong_gripper": 44
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8188669178698856,
+      "top1_action_selection": 0.684,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9759227468818652,
+      "potential_edge_mae": 0.1475369496848387,
+      "effect_prediction_mae": 0.04776298720370394,
+      "selection_regret": 0.032691791389137505,
+      "selected_candidate_type_counts": {
+        "expert": 295,
+        "near_miss": 8,
+        "wrong_direction": 7,
+        "wrong_gripper": 190
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8706779518980977,
+      "top1_action_selection": 0.498,
+      "selected_success_rate": 0.316,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9707502646546345,
+      "potential_edge_mae": 0.2958169730574824,
+      "effect_prediction_mae": 0.02143634675771565,
+      "selection_regret": 0.09696282734721899,
+      "selected_candidate_type_counts": {
+        "expert": 410,
+        "near_miss": 179,
+        "no_op": 362,
+        "wrong_direction": 27,
+        "wrong_gripper": 22
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8472046763591204,
+      "top1_action_selection": 0.706,
+      "selected_success_rate": 0.612,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9831143135233674,
+      "potential_edge_mae": 0.20503641737197578,
+      "effect_prediction_mae": 0.02327047512314227,
+      "selection_regret": 0.03612557476013899,
+      "selected_candidate_type_counts": {
+        "expert": 277,
+        "near_miss": 88,
+        "no_op": 6,
+        "random_negative": 90,
+        "wrong_direction": 36,
+        "wrong_gripper": 3
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.862627731781905,
+      "top1_action_selection": 0.688,
+      "selected_success_rate": 0.65,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9744540449221311,
+      "potential_edge_mae": 0.3605412413564324,
+      "effect_prediction_mae": 0.02591101919445702,
+      "selection_regret": 0.05477676001191139,
+      "selected_candidate_type_counts": {
+        "expert": 385,
+        "near_miss": 65,
+        "no_op": 7,
+        "random_negative": 7,
+        "wrong_direction": 32,
+        "wrong_gripper": 4
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8286214618868815,
+      "top1_action_selection": 0.544,
+      "selected_success_rate": 0.332,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9641314856950247,
+      "potential_edge_mae": 0.3129677290859356,
+      "effect_prediction_mae": 0.0357717620939687,
+      "selection_regret": 0.10844541150331498,
+      "selected_candidate_type_counts": {
+        "expert": 324,
+        "near_miss": 125,
+        "no_op": 2,
+        "wrong_direction": 24,
+        "wrong_gripper": 25
+      }
+    }
+  }
+}
+✅ Seed 1 complete
+   Success: 0.3766 | Top1: 0.5960 | Rank: 0.8470
+
+Evaluating seed 2...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a1_revised_enhanced/seed_2/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 2,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8469988586516973,
+  "top1_action_selection": 0.5908571428571429,
+  "selected_success_rate": 0.37514285714285717,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9715571386731926,
+  "potential_edge_mae": 0.2852598237931919,
+  "effect_prediction_mae": 0.027762871967903367,
+  "selection_regret": 0.07962946100586227,
+  "selected_candidate_type_counts": {
+    "expert": 2035,
+    "near_miss": 469,
+    "no_op": 445,
+    "random_negative": 46,
+    "wrong_direction": 107,
+    "wrong_gripper": 398
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8251455824160633,
+      "top1_action_selection": 0.52,
+      "selected_success_rate": 0.406,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9645054386640626,
+      "potential_edge_mae": 0.3259366197214057,
+      "effect_prediction_mae": 0.023472823263874003,
+      "selection_regret": 0.113965540766716,
+      "selected_candidate_type_counts": {
+        "expert": 326,
+        "near_miss": 88,
+        "no_op": 3,
+        "random_negative": 3,
+        "wrong_direction": 4,
+        "wrong_gripper": 76
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8153489612858048,
+      "top1_action_selection": 0.692,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.974927145263329,
+      "potential_edge_mae": 0.17851244468993016,
+      "effect_prediction_mae": 0.04580902023741111,
+      "selection_regret": 0.03166495469585061,
+      "selected_candidate_type_counts": {
+        "expert": 287,
+        "near_miss": 18,
+        "wrong_direction": 6,
+        "wrong_gripper": 189
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8718176485376686,
+      "top1_action_selection": 0.5,
+      "selected_success_rate": 0.314,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9714412659412455,
+      "potential_edge_mae": 0.29148840261455466,
+      "effect_prediction_mae": 0.020944095825184347,
+      "selection_regret": 0.09970813875645398,
+      "selected_candidate_type_counts": {
+        "expert": 469,
+        "near_miss": 138,
+        "no_op": 363,
+        "wrong_direction": 24,
+        "wrong_gripper": 6
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8521722373295078,
+      "top1_action_selection": 0.706,
+      "selected_success_rate": 0.61,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9807441941789684,
+      "potential_edge_mae": 0.22187108794733792,
+      "effect_prediction_mae": 0.022020053274346,
+      "selection_regret": 0.03959757074713707,
+      "selected_candidate_type_counts": {
+        "expert": 271,
+        "near_miss": 52,
+        "no_op": 70,
+        "random_negative": 40,
+        "wrong_direction": 22,
+        "wrong_gripper": 45
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8634668456776311,
+      "top1_action_selection": 0.686,
+      "selected_success_rate": 0.65,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.974837883365155,
+      "potential_edge_mae": 0.37560185345073566,
+      "effect_prediction_mae": 0.02527257962382685,
+      "selection_regret": 0.05547860264778137,
+      "selected_candidate_type_counts": {
+        "expert": 346,
+        "near_miss": 55,
+        "no_op": 3,
+        "random_negative": 3,
+        "wrong_direction": 36,
+        "wrong_gripper": 57
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8314502685682533,
+      "top1_action_selection": 0.532,
+      "selected_success_rate": 0.322,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9630027773583476,
+      "potential_edge_mae": 0.3091836632192614,
+      "effect_prediction_mae": 0.03587743572549566,
+      "selection_regret": 0.11728328067064285,
+      "selected_candidate_type_counts": {
+        "expert": 336,
+        "near_miss": 118,
+        "no_op": 6,
+        "wrong_direction": 15,
+        "wrong_gripper": 25
+      }
+    }
+  }
+}
+✅ Seed 2 complete
+   Success: 0.3751 | Top1: 0.5909 | Rank: 0.8470
+
+✅ All Phase A1-Revised evaluations complete!

workspace/logs/eval_enhanced_14706209_0.err

@@ -0,0 +1,22 @@
+Traceback (most recent call last):
+  File "/lustre09/project/6037638/knguy52/vla/scripts/eval_enhanced_checkpoint.py", line 173, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/eval_enhanced_checkpoint.py", line 161, in main
+    result = evaluate_enhanced_checkpoint(
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/eval_enhanced_checkpoint.py", line 147, in evaluate_enhanced_checkpoint
+    write_json(output_path, result)
+  File "/lustre09/project/6037638/knguy52/vla/dovla_cil/utils/io.py", line 23, in write_json
+    target = Path(path)
+             ^^^^^^^^^^
+  File "/cvmfs/soft.computecanada.ca/gentoo/2023/x86-64-v3/usr/lib/python3.11/pathlib.py", line 871, in __new__
+    self = cls._from_parts(args)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/cvmfs/soft.computecanada.ca/gentoo/2023/x86-64-v3/usr/lib/python3.11/pathlib.py", line 509, in _from_parts
+    drv, root, parts = self._parse_args(args)
+                       ^^^^^^^^^^^^^^^^^^^^^^
+  File "/cvmfs/soft.computecanada.ca/gentoo/2023/x86-64-v3/usr/lib/python3.11/pathlib.py", line 493, in _parse_args
+    a = os.fspath(a)
+        ^^^^^^^^^^^^
+TypeError: expected str, bytes or os.PathLike object, not dict

workspace/logs/eval_enhanced_14706209_0.out

@@ -0,0 +1,3 @@
+=== Evaluating Enhanced Model Seed 0 ===
+Checkpoint: /scratch/knguy52/dovla/experiments/cvpr_enhanced_model/seed_0/best.pt
+

workspace/logs/eval_enhanced_14706209_1.err

@@ -0,0 +1,22 @@
+Traceback (most recent call last):
+  File "/lustre09/project/6037638/knguy52/vla/scripts/eval_enhanced_checkpoint.py", line 173, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/eval_enhanced_checkpoint.py", line 161, in main
+    result = evaluate_enhanced_checkpoint(
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/eval_enhanced_checkpoint.py", line 147, in evaluate_enhanced_checkpoint
+    write_json(output_path, result)
+  File "/lustre09/project/6037638/knguy52/vla/dovla_cil/utils/io.py", line 23, in write_json
+    target = Path(path)
+             ^^^^^^^^^^
+  File "/cvmfs/soft.computecanada.ca/gentoo/2023/x86-64-v3/usr/lib/python3.11/pathlib.py", line 871, in __new__
+    self = cls._from_parts(args)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/cvmfs/soft.computecanada.ca/gentoo/2023/x86-64-v3/usr/lib/python3.11/pathlib.py", line 509, in _from_parts
+    drv, root, parts = self._parse_args(args)
+                       ^^^^^^^^^^^^^^^^^^^^^^
+  File "/cvmfs/soft.computecanada.ca/gentoo/2023/x86-64-v3/usr/lib/python3.11/pathlib.py", line 493, in _parse_args
+    a = os.fspath(a)
+        ^^^^^^^^^^^^
+TypeError: expected str, bytes or os.PathLike object, not dict

workspace/logs/eval_enhanced_14706209_1.out

@@ -0,0 +1,3 @@
+=== Evaluating Enhanced Model Seed 1 ===
+Checkpoint: /scratch/knguy52/dovla/experiments/cvpr_enhanced_model/seed_1/best.pt
+

workspace/logs/eval_enhanced_14706209_2.err

@@ -0,0 +1,22 @@
+Traceback (most recent call last):
+  File "/lustre09/project/6037638/knguy52/vla/scripts/eval_enhanced_checkpoint.py", line 173, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/eval_enhanced_checkpoint.py", line 161, in main
+    result = evaluate_enhanced_checkpoint(
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/eval_enhanced_checkpoint.py", line 147, in evaluate_enhanced_checkpoint
+    write_json(output_path, result)
+  File "/lustre09/project/6037638/knguy52/vla/dovla_cil/utils/io.py", line 23, in write_json
+    target = Path(path)
+             ^^^^^^^^^^
+  File "/cvmfs/soft.computecanada.ca/gentoo/2023/x86-64-v3/usr/lib/python3.11/pathlib.py", line 871, in __new__
+    self = cls._from_parts(args)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/cvmfs/soft.computecanada.ca/gentoo/2023/x86-64-v3/usr/lib/python3.11/pathlib.py", line 509, in _from_parts
+    drv, root, parts = self._parse_args(args)
+                       ^^^^^^^^^^^^^^^^^^^^^^
+  File "/cvmfs/soft.computecanada.ca/gentoo/2023/x86-64-v3/usr/lib/python3.11/pathlib.py", line 493, in _parse_args
+    a = os.fspath(a)
+        ^^^^^^^^^^^^
+TypeError: expected str, bytes or os.PathLike object, not dict

workspace/logs/eval_enhanced_14706209_2.out

@@ -0,0 +1,3 @@
+=== Evaluating Enhanced Model Seed 2 ===
+Checkpoint: /scratch/knguy52/dovla/experiments/cvpr_enhanced_model/seed_2/best.pt
+

workspace/logs/eval_enhanced_14706804_0.out

@@ -0,0 +1,10 @@
+=== Evaluating Enhanced Model Seed 0 ===
+Checkpoint: /scratch/knguy52/dovla/experiments/cvpr_enhanced_model/seed_0/best.pt
+
+Selected success rate: 0.3631
+Top-1 selection: 0.6291
+Oracle success: 0.4257
+
+✅ Evaluation complete for seed 0
+Selected success: 0.3631
+Top-1: 0.6291

workspace/logs/eval_enhanced_14706804_1.out

@@ -0,0 +1,10 @@
+=== Evaluating Enhanced Model Seed 1 ===
+Checkpoint: /scratch/knguy52/dovla/experiments/cvpr_enhanced_model/seed_1/best.pt
+
+Selected success rate: 0.3631
+Top-1 selection: 0.6291
+Oracle success: 0.4257
+
+✅ Evaluation complete for seed 1
+Selected success: 0.3631
+Top-1: 0.6291

workspace/logs/eval_enhanced_14706804_2.out

@@ -0,0 +1,10 @@
+=== Evaluating Enhanced Model Seed 2 ===
+Checkpoint: /scratch/knguy52/dovla/experiments/cvpr_enhanced_model/seed_2/best.pt
+
+Selected success rate: 0.3631
+Top-1 selection: 0.6291
+Oracle success: 0.4257
+
+✅ Evaluation complete for seed 2
+Selected success: 0.3631
+Top-1: 0.6291

workspace/logs/eval_hybrid_14720661_0.out

@@ -0,0 +1,8 @@
+=== Evaluating Hybrid Direct Model ===
+Seed: 0
+
+Selected success rate: 0.3831
+Top-1 selection: 0.5931
+Oracle success: 0.4257
+
+✅ Evaluation complete

workspace/logs/eval_hybrid_14720661_1.out

@@ -0,0 +1,8 @@
+=== Evaluating Hybrid Direct Model ===
+Seed: 1
+
+Selected success rate: 0.3737
+Top-1 selection: 0.6131
+Oracle success: 0.4257
+
+✅ Evaluation complete

workspace/logs/eval_hybrid_14720661_2.out

@@ -0,0 +1,8 @@
+=== Evaluating Hybrid Direct Model ===
+Seed: 2
+
+Selected success rate: 0.3663
+Top-1 selection: 0.6006
+Oracle success: 0.4257
+
+✅ Evaluation complete

workspace/logs/eval_phase_a2_14639576.out

@@ -0,0 +1,486 @@
+=== Evaluating Phase A2 Large Models ===
+
+Evaluating seed 0...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a2_large_model/seed_0/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8455677640911163,
+  "top1_action_selection": 0.5922857142857143,
+  "selected_success_rate": 0.3802857142857143,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9734645227544839,
+  "potential_edge_mae": 0.236036569314635,
+  "effect_prediction_mae": 0.026717467124909064,
+  "selection_regret": 0.07209627389215997,
+  "selected_candidate_type_counts": {
+    "expert": 1911,
+    "near_miss": 536,
+    "no_op": 457,
+    "random_negative": 102,
+    "wrong_direction": 132,
+    "wrong_gripper": 362
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8333189334900036,
+      "top1_action_selection": 0.516,
+      "selected_success_rate": 0.406,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.967029355561237,
+      "potential_edge_mae": 0.2937231201658752,
+      "effect_prediction_mae": 0.022073940630489883,
+      "selection_regret": 0.1129412483870983,
+      "selected_candidate_type_counts": {
+        "expert": 279,
+        "near_miss": 146,
+        "no_op": 1,
+        "random_negative": 6,
+        "wrong_direction": 12,
+        "wrong_gripper": 56
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8061122411550902,
+      "top1_action_selection": 0.688,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9735705815535792,
+      "potential_edge_mae": 0.11254235439014912,
+      "effect_prediction_mae": 0.04541783430588086,
+      "selection_regret": 0.03372641992941499,
+      "selected_candidate_type_counts": {
+        "expert": 311,
+        "wrong_gripper": 189
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8712058996061343,
+      "top1_action_selection": 0.513,
+      "selected_success_rate": 0.33,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9757036176487941,
+      "potential_edge_mae": 0.25102665239871225,
+      "effect_prediction_mae": 0.02073188594128252,
+      "selection_regret": 0.07792561732977629,
+      "selected_candidate_type_counts": {
+        "expert": 374,
+        "near_miss": 135,
+        "no_op": 445,
+        "wrong_direction": 45,
+        "wrong_gripper": 1
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8495171616384386,
+      "top1_action_selection": 0.704,
+      "selected_success_rate": 0.614,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.982102532828267,
+      "potential_edge_mae": 0.18490334437113579,
+      "effect_prediction_mae": 0.021206451333471026,
+      "selection_regret": 0.033426615230739115,
+      "selected_candidate_type_counts": {
+        "expert": 319,
+        "near_miss": 28,
+        "no_op": 4,
+        "random_negative": 91,
+        "wrong_direction": 16,
+        "wrong_gripper": 42
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8625158499291415,
+      "top1_action_selection": 0.682,
+      "selected_success_rate": 0.652,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9772097045095216,
+      "potential_edge_mae": 0.29795804891718364,
+      "effect_prediction_mae": 0.02397853401356292,
+      "selection_regret": 0.050853123515844346,
+      "selected_candidate_type_counts": {
+        "expert": 314,
+        "near_miss": 95,
+        "no_op": 5,
+        "random_negative": 3,
+        "wrong_direction": 36,
+        "wrong_gripper": 47
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8274259543013016,
+      "top1_action_selection": 0.53,
+      "selected_success_rate": 0.32,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9629322495312019,
+      "potential_edge_mae": 0.2587199100076356,
+      "effect_prediction_mae": 0.03288173770839239,
+      "selection_regret": 0.11787527552247047,
+      "selected_candidate_type_counts": {
+        "expert": 314,
+        "near_miss": 132,
+        "no_op": 2,
+        "random_negative": 2,
+        "wrong_direction": 23,
+        "wrong_gripper": 27
+      }
+    }
+  }
+}
+✅ Seed 0 complete
+   Success: 0.3802857142857143
+
+Evaluating seed 1...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a2_large_model/seed_1/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 1,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8601457801606958,
+  "top1_action_selection": 0.6134285714285714,
+  "selected_success_rate": 0.3797142857142857,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.975449575504389,
+  "potential_edge_mae": 0.2309461761861737,
+  "effect_prediction_mae": 0.024799873811406088,
+  "selection_regret": 0.07129849195879485,
+  "selected_candidate_type_counts": {
+    "expert": 2224,
+    "near_miss": 578,
+    "no_op": 103,
+    "random_negative": 118,
+    "wrong_direction": 187,
+    "wrong_gripper": 290
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8447754488431166,
+      "top1_action_selection": 0.574,
+      "selected_success_rate": 0.406,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9706239617760732,
+      "potential_edge_mae": 0.2652246011765074,
+      "effect_prediction_mae": 0.019311041167065165,
+      "selection_regret": 0.11269975239038467,
+      "selected_candidate_type_counts": {
+        "expert": 296,
+        "near_miss": 120,
+        "random_negative": 4,
+        "wrong_direction": 18,
+        "wrong_gripper": 62
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.823685351295475,
+      "top1_action_selection": 0.674,
+      "selected_success_rate": 0.012,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9741373922436591,
+      "potential_edge_mae": 0.10459398084650508,
+      "effect_prediction_mae": 0.043773863052136035,
+      "selection_regret": 0.03218391541205347,
+      "selected_candidate_type_counts": {
+        "expert": 272,
+        "near_miss": 22,
+        "wrong_direction": 10,
+        "wrong_gripper": 196
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8811028241012319,
+      "top1_action_selection": 0.519,
+      "selected_success_rate": 0.305,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9710455957520074,
+      "potential_edge_mae": 0.24937264457920053,
+      "effect_prediction_mae": 0.01913724916576849,
+      "selection_regret": 0.10442701446264982,
+      "selected_candidate_type_counts": {
+        "expert": 640,
+        "near_miss": 227,
+        "no_op": 89,
+        "wrong_direction": 36,
+        "wrong_gripper": 8
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8572254459028328,
+      "top1_action_selection": 0.716,
+      "selected_success_rate": 0.612,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9838359893059918,
+      "potential_edge_mae": 0.18399476665715153,
+      "effect_prediction_mae": 0.018029450088052436,
+      "selection_regret": 0.035458510018885135,
+      "selected_candidate_type_counts": {
+        "expert": 345,
+        "near_miss": 21,
+        "no_op": 12,
+        "random_negative": 92,
+        "wrong_direction": 19,
+        "wrong_gripper": 11
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8734056835981204,
+      "top1_action_selection": 0.698,
+      "selected_success_rate": 0.658,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9804412066611682,
+      "potential_edge_mae": 0.29355131632093134,
+      "effect_prediction_mae": 0.021161105101050775,
+      "selection_regret": 0.04121021604537964,
+      "selected_candidate_type_counts": {
+        "expert": 306,
+        "near_miss": 102,
+        "no_op": 1,
+        "random_negative": 16,
+        "wrong_direction": 72,
+        "wrong_gripper": 3
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8601592887571773,
+      "top1_action_selection": 0.594,
+      "selected_success_rate": 0.36,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9770172870398178,
+      "potential_edge_mae": 0.2685150524126529,
+      "effect_prediction_mae": 0.03304915894000212,
+      "selection_regret": 0.06868302091956138,
+      "selected_candidate_type_counts": {
+        "expert": 365,
+        "near_miss": 86,
+        "no_op": 1,
+        "random_negative": 6,
+        "wrong_direction": 32,
+        "wrong_gripper": 10
+      }
+    }
+  }
+}
+✅ Seed 1 complete
+   Success: 0.3797142857142857
+
+Evaluating seed 2...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a2_large_model/seed_2/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 2,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8434815910344949,
+  "top1_action_selection": 0.5922857142857143,
+  "selected_success_rate": 0.376,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9709595629536277,
+  "potential_edge_mae": 0.242036414944211,
+  "effect_prediction_mae": 0.027635291527002132,
+  "selection_regret": 0.07854472978785634,
+  "selected_candidate_type_counts": {
+    "expert": 2116,
+    "near_miss": 494,
+    "no_op": 378,
+    "random_negative": 81,
+    "wrong_direction": 145,
+    "wrong_gripper": 286
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.828670664063175,
+      "top1_action_selection": 0.55,
+      "selected_success_rate": 0.41,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9650907628622645,
+      "potential_edge_mae": 0.28116600280476817,
+      "effect_prediction_mae": 0.023020425341441705,
+      "selection_regret": 0.1072309367954731,
+      "selected_candidate_type_counts": {
+        "expert": 368,
+        "near_miss": 75,
+        "no_op": 2,
+        "random_negative": 3,
+        "wrong_direction": 3,
+        "wrong_gripper": 49
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8050285104538331,
+      "top1_action_selection": 0.682,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9726981014121375,
+      "potential_edge_mae": 0.12772678205132834,
+      "effect_prediction_mae": 0.046266445637896725,
+      "selection_regret": 0.03435434534028173,
+      "selected_candidate_type_counts": {
+        "expert": 298,
+        "near_miss": 4,
+        "wrong_direction": 4,
+        "wrong_gripper": 194
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8705438699405011,
+      "top1_action_selection": 0.504,
+      "selected_success_rate": 0.311,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9710229779636876,
+      "potential_edge_mae": 0.251372649468657,
+      "effect_prediction_mae": 0.020782845048889737,
+      "selection_regret": 0.10136570649594069,
+      "selected_candidate_type_counts": {
+        "expert": 466,
+        "near_miss": 154,
+        "no_op": 334,
+        "wrong_direction": 45,
+        "wrong_gripper": 1
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8482752713958418,
+      "top1_action_selection": 0.706,
+      "selected_success_rate": 0.612,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9809909574622121,
+      "potential_edge_mae": 0.19062138684834176,
+      "effect_prediction_mae": 0.022138593745818433,
+      "selection_regret": 0.03628189332038164,
+      "selected_candidate_type_counts": {
+        "expert": 272,
+        "near_miss": 86,
+        "no_op": 34,
+        "random_negative": 74,
+        "wrong_direction": 25,
+        "wrong_gripper": 9
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8562877601253077,
+      "top1_action_selection": 0.684,
+      "selected_success_rate": 0.65,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9738112716460825,
+      "potential_edge_mae": 0.3244631636030308,
+      "effect_prediction_mae": 0.02439411786038935,
+      "selection_regret": 0.055388800501823425,
+      "selected_candidate_type_counts": {
+        "expert": 396,
+        "near_miss": 50,
+        "no_op": 3,
+        "random_negative": 3,
+        "wrong_direction": 47,
+        "wrong_gripper": 1
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8270386771961138,
+      "top1_action_selection": 0.516,
+      "selected_success_rate": 0.328,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9620798913653245,
+      "potential_edge_mae": 0.26659241516066745,
+      "effect_prediction_mae": 0.03606176800568822,
+      "selection_regret": 0.11382571956515312,
+      "selected_candidate_type_counts": {
+        "expert": 316,
+        "near_miss": 125,
+        "no_op": 5,
+        "random_negative": 1,
+        "wrong_direction": 21,
+        "wrong_gripper": 32
+      }
+    }
+  }
+}
+✅ Seed 2 complete
+   Success: 0.376
+
+✅ All Phase A2 evaluations complete!

workspace/logs/eval_phase_a4_14647112.out

@@ -0,0 +1,1431 @@
+=== Evaluating Phase A4 All Configs (GPU) ===
+Config dir: /scratch/knguy52/dovla/experiments/phase_a4_hparam_sweep
+Dataset: /scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection
+
+Evaluating lr0.0001_h1024...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a4_hparam_sweep/lr0.0001_h1024/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8533052489424262,
+  "top1_action_selection": 0.6191428571428571,
+  "selected_success_rate": 0.38057142857142856,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9744854377624829,
+  "potential_edge_mae": 0.3301121477668643,
+  "effect_prediction_mae": 0.031717197984592736,
+  "selection_regret": 0.07116628329775163,
+  "selected_candidate_type_counts": {
+    "expert": 1963,
+    "near_miss": 629,
+    "no_op": 290,
+    "random_negative": 49,
+    "wrong_direction": 188,
+    "wrong_gripper": 381
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8393323080644882,
+      "top1_action_selection": 0.554,
+      "selected_success_rate": 0.412,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9694871715038119,
+      "potential_edge_mae": 0.37222844718282877,
+      "effect_prediction_mae": 0.02813377743123782,
+      "selection_regret": 0.10487956964969634,
+      "selected_candidate_type_counts": {
+        "expert": 289,
+        "near_miss": 116,
+        "random_negative": 2,
+        "wrong_direction": 13,
+        "wrong_gripper": 80
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8229850945346627,
+      "top1_action_selection": 0.678,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9745385858293178,
+      "potential_edge_mae": 0.24939746355367523,
+      "effect_prediction_mae": 0.04762054854080379,
+      "selection_regret": 0.034055997341871265,
+      "selected_candidate_type_counts": {
+        "expert": 283,
+        "near_miss": 15,
+        "wrong_direction": 11,
+        "wrong_gripper": 191
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8773988100226263,
+      "top1_action_selection": 0.566,
+      "selected_success_rate": 0.32,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9758736018886763,
+      "potential_edge_mae": 0.34097748786728616,
+      "effect_prediction_mae": 0.023769170602733695,
+      "selection_regret": 0.08316323102265596,
+      "selected_candidate_type_counts": {
+        "expert": 542,
+        "near_miss": 186,
+        "no_op": 233,
+        "wrong_direction": 37,
+        "wrong_gripper": 2
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8501166948590027,
+      "top1_action_selection": 0.702,
+      "selected_success_rate": 0.61,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9807683267663477,
+      "potential_edge_mae": 0.2495405717921957,
+      "effect_prediction_mae": 0.02615263636906166,
+      "selection_regret": 0.042707648448646066,
+      "selected_candidate_type_counts": {
+        "expert": 308,
+        "near_miss": 56,
+        "no_op": 55,
+        "random_negative": 36,
+        "wrong_direction": 32,
+        "wrong_gripper": 13
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8679794137390915,
+      "top1_action_selection": 0.704,
+      "selected_success_rate": 0.654,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9776504425422342,
+      "potential_edge_mae": 0.3834761563197952,
+      "effect_prediction_mae": 0.030241692979396575,
+      "selection_regret": 0.048684894561767576,
+      "selected_candidate_type_counts": {
+        "expert": 291,
+        "near_miss": 68,
+        "no_op": 1,
+        "random_negative": 11,
+        "wrong_direction": 45,
+        "wrong_gripper": 84
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.838387580191618,
+      "top1_action_selection": 0.564,
+      "selected_success_rate": 0.338,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9672063339182894,
+      "potential_edge_mae": 0.36392920469866236,
+      "effect_prediction_mae": 0.04233338936618127,
+      "selection_regret": 0.10150941103696823,
+      "selected_candidate_type_counts": {
+        "expert": 250,
+        "near_miss": 188,
+        "no_op": 1,
+        "wrong_direction": 50,
+        "wrong_gripper": 11
+      }
+    }
+  }
+}
+  ✅ Complete
+Evaluating lr0.0001_h256...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a4_hparam_sweep/lr0.0001_h256/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8546305231305698,
+  "top1_action_selection": 0.6017142857142858,
+  "selected_success_rate": 0.37942857142857145,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9749137980081676,
+  "potential_edge_mae": 0.3099070172759672,
+  "effect_prediction_mae": 0.028634389283933512,
+  "selection_regret": 0.07513375774105745,
+  "selected_candidate_type_counts": {
+    "expert": 1839,
+    "near_miss": 479,
+    "no_op": 525,
+    "random_negative": 128,
+    "wrong_direction": 115,
+    "wrong_gripper": 414
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8431684263275215,
+      "top1_action_selection": 0.564,
+      "selected_success_rate": 0.408,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9699546980612324,
+      "potential_edge_mae": 0.38448416580697453,
+      "effect_prediction_mae": 0.023410354449074546,
+      "selection_regret": 0.11005465108156204,
+      "selected_candidate_type_counts": {
+        "expert": 253,
+        "near_miss": 176,
+        "no_op": 1,
+        "random_negative": 2,
+        "wrong_direction": 32,
+        "wrong_gripper": 36
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8213511620927674,
+      "top1_action_selection": 0.672,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9736383339199919,
+      "potential_edge_mae": 0.18715317591096853,
+      "effect_prediction_mae": 0.04738139071297154,
+      "selection_regret": 0.03486216971836984,
+      "selected_candidate_type_counts": {
+        "expert": 272,
+        "near_miss": 22,
+        "wrong_direction": 13,
+        "wrong_gripper": 193
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8751110366211347,
+      "top1_action_selection": 0.485,
+      "selected_success_rate": 0.319,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9735882877455342,
+      "potential_edge_mae": 0.3345455800345687,
+      "effect_prediction_mae": 0.02196135628797628,
+      "selection_regret": 0.09737784077972174,
+      "selected_candidate_type_counts": {
+        "expert": 355,
+        "near_miss": 120,
+        "no_op": 514,
+        "wrong_direction": 10,
+        "wrong_gripper": 1
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8534141275721046,
+      "top1_action_selection": 0.706,
+      "selected_success_rate": 0.612,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9834001332561987,
+      "potential_edge_mae": 0.2551841262163316,
+      "effect_prediction_mae": 0.02148250060567993,
+      "selection_regret": 0.036189094796776775,
+      "selected_candidate_type_counts": {
+        "expert": 290,
+        "near_miss": 57,
+        "no_op": 1,
+        "random_negative": 88,
+        "wrong_direction": 23,
+        "wrong_gripper": 41
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8681472365182368,
+      "top1_action_selection": 0.728,
+      "selected_success_rate": 0.652,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.98101418247597,
+      "potential_edge_mae": 0.3613338892184103,
+      "effect_prediction_mae": 0.02592257785865129,
+      "selection_regret": 0.050353113740682603,
+      "selected_candidate_type_counts": {
+        "expert": 300,
+        "near_miss": 10,
+        "no_op": 1,
+        "random_negative": 31,
+        "wrong_direction": 22,
+        "wrong_gripper": 136
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8470087053157992,
+      "top1_action_selection": 0.572,
+      "selected_success_rate": 0.336,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.969212662852709,
+      "potential_edge_mae": 0.30829660008418086,
+      "effect_prediction_mae": 0.03832118878520358,
+      "selection_regret": 0.0997215932905674,
+      "selected_candidate_type_counts": {
+        "expert": 369,
+        "near_miss": 94,
+        "no_op": 8,
+        "random_negative": 7,
+        "wrong_direction": 15,
+        "wrong_gripper": 7
+      }
+    }
+  }
+}
+  ✅ Complete
+Evaluating lr0.0001_h512...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a4_hparam_sweep/lr0.0001_h512/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8460540338977927,
+  "top1_action_selection": 0.5917142857142857,
+  "selected_success_rate": 0.3617142857142857,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9672160590064904,
+  "potential_edge_mae": 0.2945582225588176,
+  "effect_prediction_mae": 0.03368794577856568,
+  "selection_regret": 0.09571207055183394,
+  "selected_candidate_type_counts": {
+    "expert": 2257,
+    "near_miss": 584,
+    "no_op": 51,
+    "random_negative": 112,
+    "wrong_direction": 164,
+    "wrong_gripper": 332
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8326623006341691,
+      "top1_action_selection": 0.524,
+      "selected_success_rate": 0.394,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9642711790468801,
+      "potential_edge_mae": 0.33132932179215696,
+      "effect_prediction_mae": 0.03081560057793187,
+      "selection_regret": 0.12836373773217202,
+      "selected_candidate_type_counts": {
+        "expert": 269,
+        "near_miss": 155,
+        "no_op": 1,
+        "random_negative": 2,
+        "wrong_direction": 10,
+        "wrong_gripper": 63
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.816532728667178,
+      "top1_action_selection": 0.696,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9747586613473478,
+      "potential_edge_mae": 0.18727906713804388,
+      "effect_prediction_mae": 0.05063540337178578,
+      "selection_regret": 0.03215906191244721,
+      "selected_candidate_type_counts": {
+        "expert": 302,
+        "near_miss": 7,
+        "wrong_direction": 2,
+        "wrong_gripper": 189
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8714908237660269,
+      "top1_action_selection": 0.492,
+      "selected_success_rate": 0.284,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.957668208745683,
+      "potential_edge_mae": 0.33035079169998766,
+      "effect_prediction_mae": 0.026629596806634525,
+      "selection_regret": 0.13361719042062759,
+      "selected_candidate_type_counts": {
+        "expert": 622,
+        "near_miss": 275,
+        "no_op": 42,
+        "wrong_direction": 48,
+        "wrong_gripper": 13
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.849731280645783,
+      "top1_action_selection": 0.698,
+      "selected_success_rate": 0.606,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9801882901214901,
+      "potential_edge_mae": 0.2161685740739312,
+      "effect_prediction_mae": 0.028652405746174076,
+      "selection_regret": 0.04885206139832735,
+      "selected_candidate_type_counts": {
+        "expert": 350,
+        "near_miss": 18,
+        "no_op": 6,
+        "random_negative": 96,
+        "wrong_direction": 22,
+        "wrong_gripper": 8
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8591034534198553,
+      "top1_action_selection": 0.71,
+      "selected_success_rate": 0.656,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9766712733483381,
+      "potential_edge_mae": 0.35147075808203515,
+      "effect_prediction_mae": 0.03188743274973553,
+      "selection_regret": 0.04478717613220215,
+      "selected_candidate_type_counts": {
+        "expert": 309,
+        "near_miss": 66,
+        "random_negative": 14,
+        "wrong_direction": 66,
+        "wrong_gripper": 45
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8231322298742191,
+      "top1_action_selection": 0.53,
+      "selected_success_rate": 0.298,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9592865916900153,
+      "potential_edge_mae": 0.30540548253338196,
+      "effect_prediction_mae": 0.04056558439106387,
+      "selection_regret": 0.14858807584643363,
+      "selected_candidate_type_counts": {
+        "expert": 405,
+        "near_miss": 63,
+        "no_op": 2,
+        "wrong_direction": 16,
+        "wrong_gripper": 14
+      }
+    }
+  }
+}
+  ✅ Complete
+Evaluating lr0.0003_h1024...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a4_hparam_sweep/lr0.0003_h1024/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8589263263449896,
+  "top1_action_selection": 0.6168571428571429,
+  "selected_success_rate": 0.38371428571428573,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9754980835417467,
+  "potential_edge_mae": 0.3058828205852706,
+  "effect_prediction_mae": 0.025145684622448906,
+  "selection_regret": 0.06594205010522688,
+  "selected_candidate_type_counts": {
+    "expert": 1929,
+    "near_miss": 707,
+    "no_op": 215,
+    "random_negative": 70,
+    "wrong_direction": 175,
+    "wrong_gripper": 404
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8434794629434432,
+      "top1_action_selection": 0.556,
+      "selected_success_rate": 0.408,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.969148287703904,
+      "potential_edge_mae": 0.3224464018827123,
+      "effect_prediction_mae": 0.02021031202654655,
+      "selection_regret": 0.11005249100923538,
+      "selected_candidate_type_counts": {
+        "expert": 278,
+        "near_miss": 166,
+        "no_op": 3,
+        "random_negative": 2,
+        "wrong_direction": 19,
+        "wrong_gripper": 32
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8305045183233852,
+      "top1_action_selection": 0.69,
+      "selected_success_rate": 0.012,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9773535799488441,
+      "potential_edge_mae": 0.1984737422437245,
+      "effect_prediction_mae": 0.044405310620696935,
+      "selection_regret": 0.02953313312307,
+      "selected_candidate_type_counts": {
+        "expert": 307,
+        "near_miss": 2,
+        "wrong_direction": 2,
+        "wrong_gripper": 189
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8814212687505237,
+      "top1_action_selection": 0.529,
+      "selected_success_rate": 0.317,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9743485785068674,
+      "potential_edge_mae": 0.33544318505214676,
+      "effect_prediction_mae": 0.01886101258951324,
+      "selection_regret": 0.08900124264508485,
+      "selected_candidate_type_counts": {
+        "expert": 566,
+        "near_miss": 202,
+        "no_op": 205,
+        "wrong_direction": 24,
+        "wrong_gripper": 3
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8559193199580327,
+      "top1_action_selection": 0.708,
+      "selected_success_rate": 0.616,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9804295819257342,
+      "potential_edge_mae": 0.2819488027624982,
+      "effect_prediction_mae": 0.01982430264436949,
+      "selection_regret": 0.03510657256096601,
+      "selected_candidate_type_counts": {
+        "expert": 184,
+        "near_miss": 147,
+        "no_op": 1,
+        "random_negative": 56,
+        "wrong_direction": 67,
+        "wrong_gripper": 45
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8698068173342284,
+      "top1_action_selection": 0.718,
+      "selected_success_rate": 0.66,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9795507310497178,
+      "potential_edge_mae": 0.3614990874971274,
+      "effect_prediction_mae": 0.02168413483905245,
+      "selection_regret": 0.03699221247434616,
+      "selected_candidate_type_counts": {
+        "expert": 300,
+        "near_miss": 41,
+        "no_op": 1,
+        "random_negative": 5,
+        "wrong_direction": 26,
+        "wrong_gripper": 127
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.850022731482261,
+      "top1_action_selection": 0.588,
+      "selected_success_rate": 0.356,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9733072471502868,
+      "potential_edge_mae": 0.30742171950764124,
+      "effect_prediction_mae": 0.03217370704745119,
+      "selection_regret": 0.07190745627880096,
+      "selected_candidate_type_counts": {
+        "expert": 294,
+        "near_miss": 149,
+        "no_op": 5,
+        "random_negative": 7,
+        "wrong_direction": 37,
+        "wrong_gripper": 8
+      }
+    }
+  }
+}
+  ✅ Complete
+Evaluating lr0.0003_h256...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a4_hparam_sweep/lr0.0003_h256/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8546884723821425,
+  "top1_action_selection": 0.6185714285714285,
+  "selected_success_rate": 0.3842857142857143,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9757918655852275,
+  "potential_edge_mae": 0.30116573954627207,
+  "effect_prediction_mae": 0.025571396784792488,
+  "selection_regret": 0.06712791710746076,
+  "selected_candidate_type_counts": {
+    "expert": 1744,
+    "near_miss": 602,
+    "no_op": 432,
+    "random_negative": 136,
+    "wrong_direction": 184,
+    "wrong_gripper": 402
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8431684263275215,
+      "top1_action_selection": 0.578,
+      "selected_success_rate": 0.406,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9695485770154603,
+      "potential_edge_mae": 0.3678041250593472,
+      "effect_prediction_mae": 0.020283942064171227,
+      "selection_regret": 0.11259605020284652,
+      "selected_candidate_type_counts": {
+        "expert": 280,
+        "near_miss": 129,
+        "no_op": 3,
+        "random_negative": 5,
+        "wrong_direction": 28,
+        "wrong_gripper": 55
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8232351862349528,
+      "top1_action_selection": 0.694,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9722609785099039,
+      "potential_edge_mae": 0.16588159309051775,
+      "effect_prediction_mae": 0.04346210667804156,
+      "selection_regret": 0.037038692271336916,
+      "selected_candidate_type_counts": {
+        "expert": 330,
+        "near_miss": 18,
+        "wrong_direction": 10,
+        "wrong_gripper": 142
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8777675354060169,
+      "top1_action_selection": 0.535,
+      "selected_success_rate": 0.333,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9785328165603038,
+      "potential_edge_mae": 0.3148331526974693,
+      "effect_prediction_mae": 0.01956397667920472,
+      "selection_regret": 0.07130561644583941,
+      "selected_candidate_type_counts": {
+        "expert": 390,
+        "near_miss": 166,
+        "no_op": 411,
+        "wrong_direction": 25,
+        "wrong_gripper": 8
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8370982592124703,
+      "top1_action_selection": 0.694,
+      "selected_success_rate": 0.608,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9789420686253112,
+      "potential_edge_mae": 0.2529799819984579,
+      "effect_prediction_mae": 0.01990097236313997,
+      "selection_regret": 0.04657359145581722,
+      "selected_candidate_type_counts": {
+        "expert": 133,
+        "near_miss": 150,
+        "no_op": 5,
+        "random_negative": 100,
+        "wrong_direction": 74,
+        "wrong_gripper": 38
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8698068173342284,
+      "top1_action_selection": 0.732,
+      "selected_success_rate": 0.66,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9815190380418491,
+      "potential_edge_mae": 0.37485219451968793,
+      "effect_prediction_mae": 0.022943312991103026,
+      "selection_regret": 0.036074809312820436,
+      "selected_candidate_type_counts": {
+        "expert": 257,
+        "near_miss": 43,
+        "no_op": 4,
+        "random_negative": 27,
+        "wrong_direction": 21,
+        "wrong_gripper": 148
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8514876492279715,
+      "top1_action_selection": 0.562,
+      "selected_success_rate": 0.34,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9712067637834442,
+      "potential_edge_mae": 0.31674896782432,
+      "effect_prediction_mae": 0.03328149003868241,
+      "selection_regret": 0.09500104361772538,
+      "selected_candidate_type_counts": {
+        "expert": 354,
+        "near_miss": 96,
+        "no_op": 9,
+        "random_negative": 4,
+        "wrong_direction": 26,
+        "wrong_gripper": 11
+      }
+    }
+  }
+}
+  ✅ Complete
+Evaluating lr0.0003_h512...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a4_hparam_sweep/lr0.0003_h512/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.848873390963444,
+  "top1_action_selection": 0.6065714285714285,
+  "selected_success_rate": 0.37942857142857145,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9729356993298082,
+  "potential_edge_mae": 0.2817135802084327,
+  "effect_prediction_mae": 0.027282091901553684,
+  "selection_regret": 0.07330002299270459,
+  "selected_candidate_type_counts": {
+    "expert": 2029,
+    "near_miss": 454,
+    "no_op": 408,
+    "random_negative": 115,
+    "wrong_direction": 136,
+    "wrong_gripper": 358
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.834614919389677,
+      "top1_action_selection": 0.546,
+      "selected_success_rate": 0.406,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9656286678043069,
+      "potential_edge_mae": 0.330484778162696,
+      "effect_prediction_mae": 0.021617836275652554,
+      "selection_regret": 0.11216536232829094,
+      "selected_candidate_type_counts": {
+        "expert": 382,
+        "near_miss": 77,
+        "no_op": 3,
+        "random_negative": 3,
+        "wrong_direction": 2,
+        "wrong_gripper": 33
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8157157624462302,
+      "top1_action_selection": 0.684,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9709639036788248,
+      "potential_edge_mae": 0.1802397970459396,
+      "effect_prediction_mae": 0.04605580753744575,
+      "selection_regret": 0.0350611395612359,
+      "selected_candidate_type_counts": {
+        "expert": 313,
+        "wrong_gripper": 187
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8742478840191067,
+      "top1_action_selection": 0.516,
+      "selected_success_rate": 0.329,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9761679921374685,
+      "potential_edge_mae": 0.30123000960816737,
+      "effect_prediction_mae": 0.022024564853054648,
+      "selection_regret": 0.07833491713553667,
+      "selected_candidate_type_counts": {
+        "expert": 378,
+        "near_miss": 165,
+        "no_op": 397,
+        "wrong_direction": 59,
+        "wrong_gripper": 1
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8497955163479862,
+      "top1_action_selection": 0.712,
+      "selected_success_rate": 0.608,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9807627407096544,
+      "potential_edge_mae": 0.23148925279860208,
+      "effect_prediction_mae": 0.02198729162654722,
+      "selection_regret": 0.044035196483135225,
+      "selected_candidate_type_counts": {
+        "expert": 346,
+        "near_miss": 10,
+        "no_op": 1,
+        "random_negative": 104,
+        "wrong_direction": 6,
+        "wrong_gripper": 33
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8656298948310585,
+      "top1_action_selection": 0.712,
+      "selected_success_rate": 0.652,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9773934709952478,
+      "potential_edge_mae": 0.3385302822398105,
+      "effect_prediction_mae": 0.024117636678057382,
+      "selection_regret": 0.05143456473946571,
+      "selected_candidate_type_counts": {
+        "expert": 299,
+        "near_miss": 56,
+        "no_op": 3,
+        "random_negative": 8,
+        "wrong_direction": 43,
+        "wrong_gripper": 91
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8294128542322652,
+      "top1_action_selection": 0.56,
+      "selected_success_rate": 0.322,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9634651278456848,
+      "potential_edge_mae": 0.28564565038845224,
+      "effect_prediction_mae": 0.03314694148706474,
+      "selection_regret": 0.11373406356573104,
+      "selected_candidate_type_counts": {
+        "expert": 311,
+        "near_miss": 146,
+        "no_op": 4,
+        "wrong_direction": 26,
+        "wrong_gripper": 13
+      }
+    }
+  }
+}
+  ✅ Complete
+Evaluating lr0.001_h1024...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a4_hparam_sweep/lr0.001_h1024/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8434815910344949,
+  "top1_action_selection": 0.604,
+  "selected_success_rate": 0.38057142857142856,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9730876491517635,
+  "potential_edge_mae": 0.2698472616256089,
+  "effect_prediction_mae": 0.030579289966498117,
+  "selection_regret": 0.07103479064841356,
+  "selected_candidate_type_counts": {
+    "expert": 2213,
+    "near_miss": 457,
+    "no_op": 416,
+    "random_negative": 19,
+    "wrong_direction": 92,
+    "wrong_gripper": 303
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8308306405626307,
+      "top1_action_selection": 0.54,
+      "selected_success_rate": 0.406,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9660322974903598,
+      "potential_edge_mae": 0.3137131099487799,
+      "effect_prediction_mae": 0.02394908003827207,
+      "selection_regret": 0.11257329866290093,
+      "selected_candidate_type_counts": {
+        "expert": 371,
+        "near_miss": 75,
+        "no_op": 1,
+        "random_negative": 3,
+        "wrong_direction": 3,
+        "wrong_gripper": 47
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8094801427189969,
+      "top1_action_selection": 0.688,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9733121872704474,
+      "potential_edge_mae": 0.16557472402521875,
+      "effect_prediction_mae": 0.05283002683380975,
+      "selection_regret": 0.03372641992941499,
+      "selected_candidate_type_counts": {
+        "expert": 311,
+        "wrong_gripper": 189
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8727143216290958,
+      "top1_action_selection": 0.546,
+      "selected_success_rate": 0.337,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9785930782533997,
+      "potential_edge_mae": 0.2800411258507225,
+      "effect_prediction_mae": 0.024340884746986778,
+      "selection_regret": 0.06354818376153708,
+      "selected_candidate_type_counts": {
+        "expert": 528,
+        "near_miss": 152,
+        "no_op": 301,
+        "wrong_direction": 15,
+        "wrong_gripper": 4
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8477185619767467,
+      "top1_action_selection": 0.698,
+      "selected_success_rate": 0.61,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9815710762702458,
+      "potential_edge_mae": 0.21821787461055844,
+      "effect_prediction_mae": 0.02772435398163279,
+      "selection_regret": 0.03969677586853504,
+      "selected_candidate_type_counts": {
+        "expert": 335,
+        "near_miss": 10,
+        "no_op": 99,
+        "random_negative": 8,
+        "wrong_direction": 10,
+        "wrong_gripper": 38
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8583762213768926,
+      "top1_action_selection": 0.68,
+      "selected_success_rate": 0.652,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9756178630873861,
+      "potential_edge_mae": 0.3387245252715125,
+      "effect_prediction_mae": 0.02652445964278251,
+      "selection_regret": 0.05166086229681969,
+      "selected_candidate_type_counts": {
+        "expert": 361,
+        "near_miss": 72,
+        "no_op": 2,
+        "random_negative": 7,
+        "wrong_direction": 51,
+        "wrong_gripper": 7
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8146289716950951,
+      "top1_action_selection": 0.53,
+      "selected_success_rate": 0.312,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9578939634370929,
+      "potential_edge_mae": 0.29033188505527935,
+      "effect_prediction_mae": 0.0343453397750186,
+      "selection_regret": 0.1324898102581501,
+      "selected_candidate_type_counts": {
+        "expert": 307,
+        "near_miss": 148,
+        "no_op": 13,
+        "random_negative": 1,
+        "wrong_direction": 13,
+        "wrong_gripper": 18
+      }
+    }
+  }
+}
+  ✅ Complete
+Evaluating lr0.001_h256...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a4_hparam_sweep/lr0.001_h256/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8457542095092202,
+  "top1_action_selection": 0.6085714285714285,
+  "selected_success_rate": 0.3811428571428571,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.973310139352499,
+  "potential_edge_mae": 0.27466379023723564,
+  "effect_prediction_mae": 0.027333826815782235,
+  "selection_regret": 0.07030650216580502,
+  "selected_candidate_type_counts": {
+    "expert": 2067,
+    "near_miss": 496,
+    "no_op": 392,
+    "random_negative": 98,
+    "wrong_direction": 109,
+    "wrong_gripper": 338
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8347013184496552,
+      "top1_action_selection": 0.542,
+      "selected_success_rate": 0.406,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9659351211811302,
+      "potential_edge_mae": 0.3184371621186663,
+      "effect_prediction_mae": 0.021791804283196124,
+      "selection_regret": 0.11241285344958306,
+      "selected_candidate_type_counts": {
+        "expert": 357,
+        "near_miss": 65,
+        "no_op": 1,
+        "wrong_direction": 2,
+        "wrong_gripper": 75
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8062122778352062,
+      "top1_action_selection": 0.696,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9714111646415328,
+      "potential_edge_mae": 0.16441658506721096,
+      "effect_prediction_mae": 0.047290608005391595,
+      "selection_regret": 0.03618285569734871,
+      "selected_candidate_type_counts": {
+        "expert": 325,
+        "wrong_gripper": 175
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8749182938070896,
+      "top1_action_selection": 0.551,
+      "selected_success_rate": 0.332,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9781619371359341,
+      "potential_edge_mae": 0.2922379194550684,
+      "effect_prediction_mae": 0.021420027887971895,
+      "selection_regret": 0.07034030451625585,
+      "selected_candidate_type_counts": {
+        "expert": 456,
+        "near_miss": 142,
+        "no_op": 373,
+        "wrong_direction": 25,
+        "wrong_gripper": 4
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8435004175320643,
+      "top1_action_selection": 0.706,
+      "selected_success_rate": 0.612,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9810346379461297,
+      "potential_edge_mae": 0.21957116572661076,
+      "effect_prediction_mae": 0.024783752131495196,
+      "selection_regret": 0.035909125804901125,
+      "selected_candidate_type_counts": {
+        "expert": 357,
+        "near_miss": 7,
+        "no_op": 10,
+        "random_negative": 93,
+        "wrong_direction": 14,
+        "wrong_gripper": 19
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8630006712911166,
+      "top1_action_selection": 0.702,
+      "selected_success_rate": 0.658,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9783076597528776,
+      "potential_edge_mae": 0.34548913748399146,
+      "effect_prediction_mae": 0.022947574316882033,
+      "selection_regret": 0.040340313732624054,
+      "selected_candidate_type_counts": {
+        "expert": 379,
+        "near_miss": 36,
+        "no_op": 4,
+        "random_negative": 4,
+        "wrong_direction": 41,
+        "wrong_gripper": 36
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8240583272996683,
+      "top1_action_selection": 0.512,
+      "selected_success_rate": 0.318,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9601585176739642,
+      "potential_edge_mae": 0.28740767123109495,
+      "effect_prediction_mae": 0.031682993197568544,
+      "selection_regret": 0.12661975744366646,
+      "selected_candidate_type_counts": {
+        "expert": 193,
+        "near_miss": 246,
+        "no_op": 4,
+        "random_negative": 1,
+        "wrong_direction": 27,
+        "wrong_gripper": 29
+      }
+    }
+  }
+}
+  ✅ Complete
+Evaluating lr0.001_h512...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a4_hparam_sweep/lr0.001_h512/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8457743657706369,
+  "top1_action_selection": 0.6091428571428571,
+  "selected_success_rate": 0.3822857142857143,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9737058698114338,
+  "potential_edge_mae": 0.2749793188709849,
+  "effect_prediction_mae": 0.029349568262953665,
+  "selection_regret": 0.0673150017128459,
+  "selected_candidate_type_counts": {
+    "expert": 2161,
+    "near_miss": 519,
+    "no_op": 366,
+    "random_negative": 53,
+    "wrong_direction": 89,
+    "wrong_gripper": 312
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8306578424426743,
+      "top1_action_selection": 0.534,
+      "selected_success_rate": 0.402,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9654602072077931,
+      "potential_edge_mae": 0.31413109101986825,
+      "effect_prediction_mae": 0.02357596442498993,
+      "selection_regret": 0.11722570392489433,
+      "selected_candidate_type_counts": {
+        "expert": 333,
+        "near_miss": 124,
+        "no_op": 1,
+        "random_negative": 3,
+        "wrong_direction": 8,
+        "wrong_gripper": 31
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.814481976724799,
+      "top1_action_selection": 0.688,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9731568489181334,
+      "potential_edge_mae": 0.1779422263486051,
+      "effect_prediction_mae": 0.05020761414601304,
+      "selection_regret": 0.03372641992941499,
+      "selected_candidate_type_counts": {
+        "expert": 311,
+        "wrong_gripper": 189
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8753456800469287,
+      "top1_action_selection": 0.553,
+      "selected_success_rate": 0.335,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9782237180633907,
+      "potential_edge_mae": 0.29113034868420423,
+      "effect_prediction_mae": 0.023486444911958917,
+      "selection_regret": 0.06449837126582861,
+      "selected_candidate_type_counts": {
+        "expert": 516,
+        "near_miss": 166,
+        "no_op": 291,
+        "wrong_direction": 23,
+        "wrong_gripper": 4
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8492173950281566,
+      "top1_action_selection": 0.698,
+      "selected_success_rate": 0.612,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9816916418548487,
+      "potential_edge_mae": 0.219009720069085,
+      "effect_prediction_mae": 0.0268146973231316,
+      "selection_regret": 0.03608278898149729,
+      "selected_candidate_type_counts": {
+        "expert": 297,
+        "near_miss": 20,
+        "no_op": 68,
+        "random_negative": 42,
+        "wrong_direction": 11,
+        "wrong_gripper": 62
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8605579175057806,
+      "top1_action_selection": 0.688,
+      "selected_success_rate": 0.656,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.978374739876166,
+      "potential_edge_mae": 0.3403800619725687,
+      "effect_prediction_mae": 0.025413101645652355,
+      "selection_regret": 0.04361427557468414,
+      "selected_candidate_type_counts": {
+        "expert": 403,
+        "near_miss": 47,
+        "no_op": 2,
+        "random_negative": 8,
+        "wrong_direction": 30,
+        "wrong_gripper": 10
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8166327097610668,
+      "top1_action_selection": 0.55,
+      "selected_success_rate": 0.326,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9608102146963285,
+      "potential_edge_mae": 0.2873330083330292,
+      "effect_prediction_mae": 0.03246271047697145,
+      "selection_regret": 0.11155908104777336,
+      "selected_candidate_type_counts": {
+        "expert": 301,
+        "near_miss": 162,
+        "no_op": 4,
+        "wrong_direction": 17,
+        "wrong_gripper": 16
+      }
+    }
+  }
+}
+  ✅ Complete
+
+✅ Phase A4 evaluation complete!

workspace/logs/eval_phase_a5_14623957.out

@@ -0,0 +1,637 @@
+=== Evaluating Phase A5 (Horizon Sweep) ===
+
+Evaluating H=4...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a5_horizon_sweep/h4/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.848873390963444,
+  "top1_action_selection": 0.6065714285714285,
+  "selected_success_rate": 0.37942857142857145,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9729356993298082,
+  "potential_edge_mae": 0.2817135802084327,
+  "effect_prediction_mae": 0.027282091901553684,
+  "selection_regret": 0.07330002299270459,
+  "selected_candidate_type_counts": {
+    "expert": 2029,
+    "near_miss": 454,
+    "no_op": 408,
+    "random_negative": 115,
+    "wrong_direction": 136,
+    "wrong_gripper": 358
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.834614919389677,
+      "top1_action_selection": 0.546,
+      "selected_success_rate": 0.406,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9656286678043069,
+      "potential_edge_mae": 0.330484778162696,
+      "effect_prediction_mae": 0.021617836275652554,
+      "selection_regret": 0.11216536232829094,
+      "selected_candidate_type_counts": {
+        "expert": 382,
+        "near_miss": 77,
+        "no_op": 3,
+        "random_negative": 3,
+        "wrong_direction": 2,
+        "wrong_gripper": 33
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8157157624462302,
+      "top1_action_selection": 0.684,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9709639036788248,
+      "potential_edge_mae": 0.1802397970459396,
+      "effect_prediction_mae": 0.04605580753744575,
+      "selection_regret": 0.0350611395612359,
+      "selected_candidate_type_counts": {
+        "expert": 313,
+        "wrong_gripper": 187
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8742478840191067,
+      "top1_action_selection": 0.516,
+      "selected_success_rate": 0.329,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9761679921374685,
+      "potential_edge_mae": 0.30123000960816737,
+      "effect_prediction_mae": 0.022024564853054648,
+      "selection_regret": 0.07833491713553667,
+      "selected_candidate_type_counts": {
+        "expert": 378,
+        "near_miss": 165,
+        "no_op": 397,
+        "wrong_direction": 59,
+        "wrong_gripper": 1
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8497955163479862,
+      "top1_action_selection": 0.712,
+      "selected_success_rate": 0.608,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9807627407096544,
+      "potential_edge_mae": 0.23148925279860208,
+      "effect_prediction_mae": 0.02198729162654722,
+      "selection_regret": 0.044035196483135225,
+      "selected_candidate_type_counts": {
+        "expert": 346,
+        "near_miss": 10,
+        "no_op": 1,
+        "random_negative": 104,
+        "wrong_direction": 6,
+        "wrong_gripper": 33
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8656298948310585,
+      "top1_action_selection": 0.712,
+      "selected_success_rate": 0.652,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9773934709952478,
+      "potential_edge_mae": 0.3385302822398105,
+      "effect_prediction_mae": 0.024117636678057382,
+      "selection_regret": 0.05143456473946571,
+      "selected_candidate_type_counts": {
+        "expert": 299,
+        "near_miss": 56,
+        "no_op": 3,
+        "random_negative": 8,
+        "wrong_direction": 43,
+        "wrong_gripper": 91
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8294128542322652,
+      "top1_action_selection": 0.56,
+      "selected_success_rate": 0.322,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9634651278456848,
+      "potential_edge_mae": 0.28564565038845224,
+      "effect_prediction_mae": 0.03314694148706474,
+      "selection_regret": 0.11373406356573104,
+      "selected_candidate_type_counts": {
+        "expert": 311,
+        "near_miss": 146,
+        "no_op": 4,
+        "wrong_direction": 26,
+        "wrong_gripper": 13
+      }
+    }
+  }
+}
+✅ H=4 complete
+
+Evaluating H=8...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a5_horizon_sweep/h8/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8485483712481009,
+  "top1_action_selection": 0.5965714285714285,
+  "selected_success_rate": 0.3802857142857143,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9725010947905504,
+  "potential_edge_mae": 0.28866785881384577,
+  "effect_prediction_mae": 0.028066217648649663,
+  "selection_regret": 0.07148465641428317,
+  "selected_candidate_type_counts": {
+    "expert": 2054,
+    "near_miss": 551,
+    "no_op": 403,
+    "random_negative": 83,
+    "wrong_direction": 113,
+    "wrong_gripper": 296
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8325586217621952,
+      "top1_action_selection": 0.536,
+      "selected_success_rate": 0.404,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9647240884995278,
+      "potential_edge_mae": 0.32863840987709747,
+      "effect_prediction_mae": 0.023600924187944353,
+      "selection_regret": 0.11502571770548821,
+      "selected_candidate_type_counts": {
+        "expert": 360,
+        "near_miss": 100,
+        "no_op": 1,
+        "random_negative": 1,
+        "wrong_direction": 2,
+        "wrong_gripper": 36
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8233352229150689,
+      "top1_action_selection": 0.684,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.971779476162129,
+      "potential_edge_mae": 0.20254382133708312,
+      "effect_prediction_mae": 0.04473089839226843,
+      "selection_regret": 0.034455125350505114,
+      "selected_candidate_type_counts": {
+        "expert": 309,
+        "wrong_gripper": 191
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8731836084806838,
+      "top1_action_selection": 0.526,
+      "selected_success_rate": 0.332,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9766583483581768,
+      "potential_edge_mae": 0.307460764325826,
+      "effect_prediction_mae": 0.023161722374274427,
+      "selection_regret": 0.07190046679973602,
+      "selected_candidate_type_counts": {
+        "expert": 408,
+        "near_miss": 176,
+        "no_op": 378,
+        "wrong_direction": 36,
+        "wrong_gripper": 2
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8521080016273045,
+      "top1_action_selection": 0.702,
+      "selected_success_rate": 0.606,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9803024257139668,
+      "potential_edge_mae": 0.2296969227850877,
+      "effect_prediction_mae": 0.022678795472643554,
+      "selection_regret": 0.047168828047811986,
+      "selected_candidate_type_counts": {
+        "expert": 322,
+        "near_miss": 40,
+        "no_op": 15,
+        "random_negative": 78,
+        "wrong_direction": 22,
+        "wrong_gripper": 23
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8645297232788841,
+      "top1_action_selection": 0.684,
+      "selected_success_rate": 0.656,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9764559542656174,
+      "potential_edge_mae": 0.3461657687938562,
+      "effect_prediction_mae": 0.024554847819109406,
+      "selection_regret": 0.04448705795407295,
+      "selected_candidate_type_counts": {
+        "expert": 374,
+        "near_miss": 67,
+        "no_op": 7,
+        "random_negative": 4,
+        "wrong_direction": 33,
+        "wrong_gripper": 15
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8228628197140885,
+      "top1_action_selection": 0.518,
+      "selected_success_rate": 0.322,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9609290221762656,
+      "potential_edge_mae": 0.2933905524675264,
+      "effect_prediction_mae": 0.034574612920032624,
+      "selection_regret": 0.11545493224263191,
+      "selected_candidate_type_counts": {
+        "expert": 281,
+        "near_miss": 168,
+        "no_op": 2,
+        "wrong_direction": 20,
+        "wrong_gripper": 29
+      }
+    }
+  }
+}
+✅ H=8 complete
+
+Evaluating H=12...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a5_horizon_sweep/h12/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8513979627058773,
+  "top1_action_selection": 0.6137142857142858,
+  "selected_success_rate": 0.38171428571428573,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9739879618625659,
+  "potential_edge_mae": 0.311519485185442,
+  "effect_prediction_mae": 0.025371263522738295,
+  "selection_regret": 0.06960772782510945,
+  "selected_candidate_type_counts": {
+    "expert": 2172,
+    "near_miss": 545,
+    "no_op": 226,
+    "random_negative": 51,
+    "wrong_direction": 124,
+    "wrong_gripper": 382
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.8372760104370064,
+      "top1_action_selection": 0.534,
+      "selected_success_rate": 0.402,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.9666355424432421,
+      "potential_edge_mae": 0.32322976398370507,
+      "effect_prediction_mae": 0.02049270931245466,
+      "selection_regret": 0.11816497150063515,
+      "selected_candidate_type_counts": {
+        "expert": 332,
+        "near_miss": 125,
+        "no_op": 2,
+        "random_negative": 3,
+        "wrong_direction": 7,
+        "wrong_gripper": 31
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8275867818200007,
+      "top1_action_selection": 0.688,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9756686311964435,
+      "potential_edge_mae": 0.20839558574568923,
+      "effect_prediction_mae": 0.04319144305446601,
+      "selection_regret": 0.03323618159070611,
+      "selected_candidate_type_counts": {
+        "expert": 305,
+        "near_miss": 3,
+        "wrong_direction": 2,
+        "wrong_gripper": 190
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8782452023799547,
+      "top1_action_selection": 0.545,
+      "selected_success_rate": 0.328,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9766781829196756,
+      "potential_edge_mae": 0.33982576701873135,
+      "effect_prediction_mae": 0.019926206135357987,
+      "selection_regret": 0.07382571572810412,
+      "selected_candidate_type_counts": {
+        "expert": 586,
+        "near_miss": 173,
+        "no_op": 214,
+        "wrong_direction": 27
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8509731708883798,
+      "top1_action_selection": 0.708,
+      "selected_success_rate": 0.612,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9819373152379046,
+      "potential_edge_mae": 0.25320796231382414,
+      "effect_prediction_mae": 0.019695551005677287,
+      "selection_regret": 0.036286431349813936,
+      "selected_candidate_type_counts": {
+        "expert": 308,
+        "near_miss": 72,
+        "no_op": 2,
+        "random_negative": 36,
+        "wrong_direction": 48,
+        "wrong_gripper": 34
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.861117326769598,
+      "top1_action_selection": 0.724,
+      "selected_success_rate": 0.656,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9790737222751167,
+      "potential_edge_mae": 0.35681431011837317,
+      "effect_prediction_mae": 0.021291259668266945,
+      "selection_regret": 0.04376586377620697,
+      "selected_candidate_type_counts": {
+        "expert": 349,
+        "near_miss": 16,
+        "no_op": 1,
+        "random_negative": 5,
+        "wrong_direction": 10,
+        "wrong_gripper": 119
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8268197814410075,
+      "top1_action_selection": 0.552,
+      "selected_success_rate": 0.336,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9612441560458905,
+      "potential_edge_mae": 0.352334169154979,
+      "effect_prediction_mae": 0.0330754693475864,
+      "selection_regret": 0.10814921510219574,
+      "selected_candidate_type_counts": {
+        "expert": 292,
+        "near_miss": 156,
+        "no_op": 7,
+        "random_negative": 7,
+        "wrong_direction": 30,
+        "wrong_gripper": 8
+      }
+    }
+  }
+}
+✅ H=12 complete
+
+Evaluating H=16...
+{
+  "checkpoint": "/scratch/knguy52/dovla/experiments/phase_a5_horizon_sweep/h16/best.pt",
+  "dataset": "/scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection",
+  "split": "all_groups",
+  "seed": 0,
+  "k": 16,
+  "training_k": null,
+  "evaluation_k": 16,
+  "objective": "lattice_field",
+  "observation_mode": "state",
+  "backbone_type": "native",
+  "backbone_model": null,
+  "val_fraction": 0.2,
+  "num_groups": 3500,
+  "num_records": 56000,
+  "num_pairs": 396899,
+  "pairwise_ranking_accuracy": 0.8454317093265541,
+  "top1_action_selection": 0.606,
+  "selected_success_rate": 0.374,
+  "oracle_success_rate": 0.4257142857142857,
+  "ndcg_at_k": 0.9723573627528053,
+  "potential_edge_mae": 0.3004329855448288,
+  "effect_prediction_mae": 0.02708220269034668,
+  "selection_regret": 0.07832444343449814,
+  "selected_candidate_type_counts": {
+    "expert": 2309,
+    "near_miss": 507,
+    "no_op": 164,
+    "random_negative": 56,
+    "wrong_direction": 106,
+    "wrong_gripper": 358
+  },
+  "per_task": {
+    "LiftPegUpright-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 57871,
+      "pairwise_ranking_accuracy": 0.826718045307667,
+      "top1_action_selection": 0.534,
+      "selected_success_rate": 0.394,
+      "oracle_success_rate": 0.492,
+      "ndcg_at_k": 0.964069332117036,
+      "potential_edge_mae": 0.3301231290942507,
+      "effect_prediction_mae": 0.023808806435093702,
+      "selection_regret": 0.12785429334640502,
+      "selected_candidate_type_counts": {
+        "expert": 364,
+        "near_miss": 93,
+        "random_negative": 2,
+        "wrong_direction": 5,
+        "wrong_gripper": 36
+      }
+    },
+    "PegInsertionSide-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59978,
+      "pairwise_ranking_accuracy": 0.8099303077795191,
+      "top1_action_selection": 0.684,
+      "selected_success_rate": 0.01,
+      "oracle_success_rate": 0.026,
+      "ndcg_at_k": 0.9732681459128515,
+      "potential_edge_mae": 0.20445765791515638,
+      "effect_prediction_mae": 0.047586079942799946,
+      "selection_regret": 0.034455125350505114,
+      "selected_candidate_type_counts": {
+        "expert": 309,
+        "wrong_gripper": 191
+      }
+    },
+    "PickCube-v1": {
+      "num_groups": 1000,
+      "num_records": 16000,
+      "num_pairs": 119330,
+      "pairwise_ranking_accuracy": 0.8753875806586776,
+      "top1_action_selection": 0.545,
+      "selected_success_rate": 0.322,
+      "oracle_success_rate": 0.374,
+      "ndcg_at_k": 0.9750336101290297,
+      "potential_edge_mae": 0.3430605992787838,
+      "effect_prediction_mae": 0.020885566346230018,
+      "selection_regret": 0.07967505978792906,
+      "selected_candidate_type_counts": {
+        "expert": 613,
+        "near_miss": 206,
+        "no_op": 149,
+        "wrong_direction": 26,
+        "wrong_gripper": 6
+      }
+    },
+    "PullCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 46703,
+      "pairwise_ranking_accuracy": 0.8414448750615592,
+      "top1_action_selection": 0.696,
+      "selected_success_rate": 0.606,
+      "oracle_success_rate": 0.628,
+      "ndcg_at_k": 0.9809098310104648,
+      "potential_edge_mae": 0.21708434699290485,
+      "effect_prediction_mae": 0.02073788793229813,
+      "selection_regret": 0.04757944610714913,
+      "selected_candidate_type_counts": {
+        "expert": 276,
+        "near_miss": 34,
+        "no_op": 9,
+        "random_negative": 39,
+        "wrong_direction": 36,
+        "wrong_gripper": 106
+      }
+    },
+    "PushCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 53628,
+      "pairwise_ranking_accuracy": 0.8619377936898635,
+      "top1_action_selection": 0.698,
+      "selected_success_rate": 0.654,
+      "oracle_success_rate": 0.678,
+      "ndcg_at_k": 0.9775019818027816,
+      "potential_edge_mae": 0.3419758065459811,
+      "effect_prediction_mae": 0.023185871818971894,
+      "selection_regret": 0.04735343900322914,
+      "selected_candidate_type_counts": {
+        "expert": 400,
+        "near_miss": 49,
+        "no_op": 1,
+        "random_negative": 14,
+        "wrong_direction": 24,
+        "wrong_gripper": 12
+      }
+    },
+    "StackCube-v1": {
+      "num_groups": 500,
+      "num_records": 8000,
+      "num_pairs": 59389,
+      "pairwise_ranking_accuracy": 0.8275606593813669,
+      "top1_action_selection": 0.54,
+      "selected_success_rate": 0.31,
+      "oracle_success_rate": 0.408,
+      "ndcg_at_k": 0.9606850281684548,
+      "potential_edge_mae": 0.31080914641623364,
+      "effect_prediction_mae": 0.03248564001080156,
+      "selection_regret": 0.13167868065834046,
+      "selected_candidate_type_counts": {
+        "expert": 347,
+        "near_miss": 125,
+        "no_op": 5,
+        "random_negative": 1,
+        "wrong_direction": 15,
+        "wrong_gripper": 7
+      }
+    }
+  }
+}
+✅ H=16 complete
+
+✅ All Phase A5 evaluations complete!

workspace/logs/eval_transformer_14708976_0.out

@@ -0,0 +1,8 @@
+=== Evaluating Baseline Transformer (No Language) ===
+Seed: 0
+
+Selected success rate: 0.3780
+Top-1 selection: 0.6429
+Oracle success: 0.4257
+
+✅ Evaluation complete

workspace/logs/eval_transformer_14708976_1.err

@@ -0,0 +1,43 @@
+Traceback (most recent call last):
+  File "/lustre09/project/6037638/knguy52/vla/scripts/eval_transformer_checkpoint.py", line 155, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/eval_transformer_checkpoint.py", line 143, in main
+    result = evaluate_transformer_checkpoint(
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/eval_transformer_checkpoint.py", line 36, in evaluate_transformer_checkpoint
+    checkpoint = torch.load(checkpoint_path, map_location=resolved_device, weights_only=False)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/torch/serialization.py", line 1579, in load
+    return _load(
+           ^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/torch/serialization.py", line 2190, in _load
+    result = unpickler.load()
+             ^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/torch/serialization.py", line 2154, in persistent_load
+    typed_storage = load_tensor(
+                    ^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/torch/serialization.py", line 2116, in load_tensor
+    wrap_storage = restore_location(storage, location)
+                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/torch/serialization.py", line 1915, in restore_location
+    return default_restore_location(storage, map_location)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/torch/serialization.py", line 734, in default_restore_location
+    result = fn(storage, location)
+             ^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/torch/serialization.py", line 668, in _deserialize
+    return obj.to(device=device)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/torch/storage.py", line 289, in to
+    return _to(self, device, non_blocking)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/torch/_utils.py", line 106, in _to
+    untyped_storage = torch.UntypedStorage(self.size(), device=device)
+                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+torch.AcceleratorError: CUDA error: CUDA-capable device(s) is/are busy or unavailable
+Search for `cudaErrorDevicesUnavailable' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
+CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
+For debugging consider passing CUDA_LAUNCH_BLOCKING=1
+Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
+

workspace/logs/eval_transformer_14708976_1.out

@@ -0,0 +1,3 @@
+=== Evaluating Baseline Transformer (No Language) ===
+Seed: 1
+

workspace/logs/eval_transformer_14708976_2.out

@@ -0,0 +1,8 @@
+=== Evaluating Baseline Transformer (No Language) ===
+Seed: 2
+
+Selected success rate: 0.3631
+Top-1 selection: 0.6277
+Oracle success: 0.4257
+
+✅ Evaluation complete

workspace/logs/gen_embeddings_14708990.err

@@ -0,0 +1,80 @@
+'[Errno 101] Network is unreachable' thrown while requesting HEAD https://huggingface.co/sentence-transformers/all-mpnet-base-v2/resolve/main/adapter_config.json
+Retrying in 1s [Retry 1/5].
+Traceback (most recent call last):
+  File "/lustre09/project/6037638/knguy52/vla/scripts/generate_instruction_embeddings.py", line 102, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/generate_instruction_embeddings.py", line 74, in main
+    embedder = LanguageEmbedder(
+               ^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/dovla_cil/utils/language_embeddings.py", line 27, in __init__
+    self.model = SentenceTransformer(model_name)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/sentence_transformers/util/decorators.py", line 41, in wrapper
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/sentence_transformers/sentence_transformer/model.py", line 188, in __init__
+    super().__init__(
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/sentence_transformers/base/model.py", line 216, in __init__
+    modules, self.module_kwargs = self._load_modules(
+                                  ^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/sentence_transformers/base/model.py", line 1002, in _load_modules
+    return self._load_config_modules(model_name_or_path, **load_kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/sentence_transformers/base/model.py", line 1208, in _load_config_modules
+    module = module_class.load(
+             ^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/sentence_transformers/base/modules/transformer.py", line 2030, in load
+    return cls(model_name_or_path=model_name_or_path, **init_kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/sentence_transformers/util/decorators.py", line 87, in wrapper
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/sentence_transformers/base/modules/transformer.py", line 639, in __init__
+    config, is_peft_model = self._load_config(model_name_or_path, backend, config_kwargs)
+                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/sentence_transformers/base/modules/transformer.py", line 1647, in _load_config
+    adapter_config_file = find_adapter_config_file(
+                          ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/transformers/utils/peft_utils.py", line 84, in find_adapter_config_file
+    adapter_cached_filename = cached_file(
+                              ^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/transformers/utils/hub.py", line 293, in cached_file
+    file = cached_files(path_or_repo_id=path_or_repo_id, filenames=[filename], **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/transformers/utils/hub.py", line 527, in cached_files
+    raise e
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/transformers/utils/hub.py", line 437, in cached_files
+    hf_hub_download(
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 88, in _inner_fn
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1016, in hf_hub_download
+    return _hf_hub_download_to_cache_dir(
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1149, in _hf_hub_download_to_cache_dir
+    _get_metadata_or_catch_error(
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1691, in _get_metadata_or_catch_error
+    metadata = get_hf_file_metadata(
+               ^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 88, in _inner_fn
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1613, in get_hf_file_metadata
+    response = _httpx_follow_relative_redirects_with_backoff(
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/utils/_http.py", line 696, in _httpx_follow_relative_redirects_with_backoff
+    response = http_backoff(
+               ^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/utils/_http.py", line 570, in http_backoff
+    return next(
+           ^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/utils/_http.py", line 478, in _http_backoff_base
+    response = client.request(method=method, url=url, **kwargs)
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/httpx/_client.py", line 825, in request
+    return self.send(request, auth=auth, follow_redirects=follow_redirects)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/httpx/_client.py", line 901, in send
+    raise RuntimeError("Cannot send a request, as the client has been closed.")
+RuntimeError: Cannot send a request, as the client has been closed.

workspace/logs/gen_embeddings_14708990.out

@@ -0,0 +1,14 @@
+=== Generating Instruction Embeddings (Fast Parallel) ===
+Using 8 CPU cores for parallel encoding
+
+======================================================================
+Instruction Embedding Generation
+======================================================================
+Dataset: /scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection
+Output: /scratch/knguy52/dovla/experiments/instruction_embeddings.pkl
+Model: all-mpnet-base-v2
+
+Loading dataset...
+Found 3500 groups
+
+Loading embedding model: all-mpnet-base-v2

workspace/logs/hybrid_direct_14714365_0.err

@@ -0,0 +1,11 @@
+Traceback (most recent call last):
+  File "/lustre09/project/6037638/knguy52/vla/scripts/train_hybrid_direct.py", line 347, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/train_hybrid_direct.py", line 286, in main
+    train_metrics = train_epoch(model, train_loader, optimizer, device)
+                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/train_hybrid_direct.py", line 149, in train_epoch
+    reward_loss = F.mse_loss(pred_rewards, target_rewards)
+                  ^
+NameError: name 'F' is not defined

workspace/logs/hybrid_direct_14714365_0.out

@@ -0,0 +1,32 @@
+= = = = = = = = = = = = = = =
+DoVLA-Hybrid: DIRECT Scoring (FIXED!)
+= = = = = = = = = = = = = = =
+
+KEY IMPROVEMENT:
+  OLD: Pairwise ranking → aggregate → 37%
+  NEW: Direct scoring → 45-48%
+
+Approach:
+  - Predict reward(action) directly
+  - Predict success(action) directly
+  - Select: argmax(success_prob * reward)
+
+Expected: 45-48% WITHOUT language
+Then +language: 55-60% final
+
+Seed: 0
+
+======================================================================
+DoVLA-Hybrid: DIRECT Scoring (NOT Pairwise)
+======================================================================
+Dataset: /scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection
+Device: cuda
+Approach: Predict reward + success DIRECTLY
+Expected: 45-48% (vs 37% pairwise baseline)
+
+Total: 3500, Train: 2800, Val: 700
+
+Model parameters: 5,093,890
+
+Starting training...
+

workspace/logs/hybrid_direct_14714365_1.err

@@ -0,0 +1,11 @@
+Traceback (most recent call last):
+  File "/lustre09/project/6037638/knguy52/vla/scripts/train_hybrid_direct.py", line 347, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/train_hybrid_direct.py", line 286, in main
+    train_metrics = train_epoch(model, train_loader, optimizer, device)
+                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/train_hybrid_direct.py", line 149, in train_epoch
+    reward_loss = F.mse_loss(pred_rewards, target_rewards)
+                  ^
+NameError: name 'F' is not defined

workspace/logs/hybrid_direct_14714365_1.out

@@ -0,0 +1,32 @@
+= = = = = = = = = = = = = = =
+DoVLA-Hybrid: DIRECT Scoring (FIXED!)
+= = = = = = = = = = = = = = =
+
+KEY IMPROVEMENT:
+  OLD: Pairwise ranking → aggregate → 37%
+  NEW: Direct scoring → 45-48%
+
+Approach:
+  - Predict reward(action) directly
+  - Predict success(action) directly
+  - Select: argmax(success_prob * reward)
+
+Expected: 45-48% WITHOUT language
+Then +language: 55-60% final
+
+Seed: 1
+
+======================================================================
+DoVLA-Hybrid: DIRECT Scoring (NOT Pairwise)
+======================================================================
+Dataset: /scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection
+Device: cuda
+Approach: Predict reward + success DIRECTLY
+Expected: 45-48% (vs 37% pairwise baseline)
+
+Total: 3500, Train: 2800, Val: 700
+
+Model parameters: 5,093,890
+
+Starting training...
+

workspace/logs/hybrid_direct_14714365_2.err

@@ -0,0 +1,11 @@
+Traceback (most recent call last):
+  File "/lustre09/project/6037638/knguy52/vla/scripts/train_hybrid_direct.py", line 347, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/train_hybrid_direct.py", line 286, in main
+    train_metrics = train_epoch(model, train_loader, optimizer, device)
+                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/lustre09/project/6037638/knguy52/vla/scripts/train_hybrid_direct.py", line 149, in train_epoch
+    reward_loss = F.mse_loss(pred_rewards, target_rewards)
+                  ^
+NameError: name 'F' is not defined

workspace/logs/hybrid_direct_14714365_2.out

@@ -0,0 +1,32 @@
+= = = = = = = = = = = = = = =
+DoVLA-Hybrid: DIRECT Scoring (FIXED!)
+= = = = = = = = = = = = = = =
+
+KEY IMPROVEMENT:
+  OLD: Pairwise ranking → aggregate → 37%
+  NEW: Direct scoring → 45-48%
+
+Approach:
+  - Predict reward(action) directly
+  - Predict success(action) directly
+  - Select: argmax(success_prob * reward)
+
+Expected: 45-48% WITHOUT language
+Then +language: 55-60% final
+
+Seed: 2
+
+======================================================================
+DoVLA-Hybrid: DIRECT Scoring (NOT Pairwise)
+======================================================================
+Dataset: /scratch/knguy52/dovla/experiments/maniskill_presuccess_six_task_collection
+Device: cuda
+Approach: Predict reward + success DIRECTLY
+Expected: 45-48% (vs 37% pairwise baseline)
+
+Total: 3500, Train: 2800, Val: 700
+
+Model parameters: 5,093,890
+
+Starting training...
+