Auto-sync: 2026-06-28 00:54:07 (part 3)
Browse files- results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_v2_summary.md +19 -0
- results/paper_core_results.md +12 -8
- results/paper_story_memo.md +19 -21
- results/paper_table_status.json +14 -54
- results/paper_table_status.md +4 -8
- scripts/build_paper_table_status.py +24 -22
- scripts/export_field_selected_policy_targets.py +159 -0
- scripts/slurm/train_dovla_h16_policy_ckpt.sbatch +5 -0
- scripts/train_dovla.py +8 -0
- tests/test_trainer.py +37 -0
results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_v2_summary.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# h=16 Best-Policy Checkpoint Rollout
|
| 2 |
+
|
| 3 |
+
Run root: `/scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs`
|
| 4 |
+
Objective: `near_miss_policy_bc5`
|
| 5 |
+
Result file: `retrieval_residual_v2_rollout.json`
|
| 6 |
+
Completed seeds: 3
|
| 7 |
+
Baseline h=4 policy success: 29.67%
|
| 8 |
+
Baseline h=16 rank-checkpoint success: 29.74%
|
| 9 |
+
|
| 10 |
+
Mean success: 32.12% +/- 1.26%
|
| 11 |
+
Gain vs h=16 rank checkpoint: +2.38%
|
| 12 |
+
Mean progress: 54.83%
|
| 13 |
+
Mean action MSE to best: 0.559
|
| 14 |
+
|
| 15 |
+
| seed | mode | k | retrieval K | sigma | opt steps | trust | success | progress | oracle | action MSE |
|
| 16 |
+
|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
| 17 |
+
| 0 | retrieval_residual | 16 | 1 | 0.00 | 0 | 0.00 | 31.48% | 53.24% | 85.74% | 0.633 |
|
| 18 |
+
| 1 | retrieval_residual | 16 | 1 | 0.00 | 0 | 0.00 | 31.30% | 54.83% | 86.96% | 0.508 |
|
| 19 |
+
| 2 | retrieval_residual | 16 | 1 | 0.00 | 0 | 0.00 | 33.57% | 56.41% | 87.65% | 0.538 |
|
results/paper_core_results.md
CHANGED
|
@@ -18,7 +18,9 @@ baseline is the h=16 rank-checkpoint online rollout (`29.74%`).
|
|
| 18 |
| Trust-region field optimization | No | No | 25.39% | -4.35 pp | Differentiable field ascent is a negative diagnostic; the field is not a generic action optimizer |
|
| 19 |
| Best non-expert proposal policy | No | No | 27.88% | -1.86 pp | Broadening BC targets beyond near-miss does not solve proposal generation |
|
| 20 |
| Best non-expert proposal + field | No | No | 26.49% | -3.25 pp | The field still needs local counterfactual proposal geometry |
|
| 21 |
-
| Train-state residual retrieval | No | No |
|
|
|
|
|
|
|
| 22 |
| Lattice, no expert/no near-miss | Yes | No | 25.57% | -4.17 pp | Non-local negatives do not help |
|
| 23 |
| Lattice, near-miss only | Yes | No | 55.94% | +26.20 pp | Local counterfactual proposals carry the gain |
|
| 24 |
| Lattice, no expert | Yes | No | 56.99% | +27.25 pp | Reviewer-safe main result |
|
|
@@ -33,15 +35,17 @@ Suggested main-table rows:
|
|
| 33 |
4. Near-miss proposal + field, BC x5 field checkpoint
|
| 34 |
5. Trust-region field optimization
|
| 35 |
6. Best non-expert proposal + field
|
| 36 |
-
7.
|
| 37 |
-
8. Lattice,
|
| 38 |
-
9. Lattice,
|
| 39 |
-
10.
|
|
|
|
| 40 |
|
| 41 |
Suggested claim:
|
| 42 |
|
| 43 |
> DoVLA-CIL is not a better behavior-cloning policy; it is a local counterfactual action
|
| 44 |
> selection rule. A deployment-clean near-miss proposal policy plus the field gives a small
|
| 45 |
-
> gain,
|
| 46 |
-
>
|
| 47 |
-
> the
|
|
|
|
|
|
| 18 |
| Trust-region field optimization | No | No | 25.39% | -4.35 pp | Differentiable field ascent is a negative diagnostic; the field is not a generic action optimizer |
|
| 19 |
| Best non-expert proposal policy | No | No | 27.88% | -1.86 pp | Broadening BC targets beyond near-miss does not solve proposal generation |
|
| 20 |
| Best non-expert proposal + field | No | No | 26.49% | -3.25 pp | The field still needs local counterfactual proposal geometry |
|
| 21 |
+
| Train-state residual retrieval | No | No | 32.12% | +2.38 pp | Transferred counterfactual residuals are a positive clean bridge but do not beat the near-miss proposal policy |
|
| 22 |
+
| KNN train-state residual retrieval | No | No | 29.91% | +0.17 pp | Adding more retrieved tangent neighborhoods dilutes the signal |
|
| 23 |
+
| Train-state near-miss residual retrieval | No | No | 14.06% smoke | -15.68 pp | Restricting to transferred near-miss residuals failed in smoke; full jobs canceled |
|
| 24 |
| Lattice, no expert/no near-miss | Yes | No | 25.57% | -4.17 pp | Non-local negatives do not help |
|
| 25 |
| Lattice, near-miss only | Yes | No | 55.94% | +26.20 pp | Local counterfactual proposals carry the gain |
|
| 26 |
| Lattice, no expert | Yes | No | 56.99% | +27.25 pp | Reviewer-safe main result |
|
|
|
|
| 35 |
4. Near-miss proposal + field, BC x5 field checkpoint
|
| 36 |
5. Trust-region field optimization
|
| 37 |
6. Best non-expert proposal + field
|
| 38 |
+
7. Train-state residual retrieval
|
| 39 |
+
8. Lattice, near-miss only
|
| 40 |
+
9. Lattice, no expert
|
| 41 |
+
10. Lattice, full
|
| 42 |
+
11. Oracle ceiling
|
| 43 |
|
| 44 |
Suggested claim:
|
| 45 |
|
| 46 |
> DoVLA-CIL is not a better behavior-cloning policy; it is a local counterfactual action
|
| 47 |
> selection rule. A deployment-clean near-miss proposal policy plus the field gives a small
|
| 48 |
+
> gain, and transferred counterfactual residuals nearly match it, while field-gradient ascent,
|
| 49 |
+
> KNN residual retrieval, and broader non-expert BC targets fail. The large effect appears only
|
| 50 |
+
> when the field is queried on same-state intervention proposals, and the mechanism is isolated
|
| 51 |
+
> to near-miss counterfactuals.
|
results/paper_story_memo.md
CHANGED
|
@@ -62,7 +62,7 @@ test-time search. The cleaner novelty is:
|
|
| 62 |
|
| 63 |
## Active Jobs
|
| 64 |
|
| 65 |
-
Last checked: `2026-06-28 04:
|
| 66 |
|
| 67 |
- `14842523`: GPU smoke for `selection_mode=field_optim`.
|
| 68 |
- `14842557`: low-resource CPU unit smoke for the pure action-optimization helper.
|
|
@@ -83,26 +83,24 @@ Last checked: `2026-06-28 04:44 UTC`.
|
|
| 83 |
- `14842619`: KNN4 `retrieval_residual` summary.
|
| 84 |
- `14842646`: CPU unit smoke for the KNN residual helper. Completed.
|
| 85 |
- `14857111`: fixed nearest-1 `retrieval_residual` smoke.
|
| 86 |
-
- `14857112`: fixed nearest-1 `retrieval_residual` full rollout.
|
| 87 |
-
- `14857113`: fixed nearest-1 `retrieval_residual` summary.
|
| 88 |
-
- `14857114`: fixed KNN4 `retrieval_residual` smoke.
|
| 89 |
-
- `14857115`: fixed KNN4 `retrieval_residual` full rollout.
|
| 90 |
-
- `14857116`: fixed KNN4 `retrieval_residual` summary.
|
| 91 |
-
- `14857117`: rebuild `paper_table_status.*` after fixed residual summaries.
|
| 92 |
-
- `14857692`: smoke nearest-1 transferred near-miss residual retrieval.
|
| 93 |
-
- `14857693`: full nearest-1 transferred near-miss residual retrieval.
|
| 94 |
-
- `14857694`: summary nearest-1 transferred near-miss residual retrieval.
|
| 95 |
-
- `14857695`: smoke KNN4 transferred near-miss residual retrieval.
|
| 96 |
-
- `14857696`: full KNN4 transferred near-miss residual retrieval.
|
| 97 |
-
- `14857697`: summary KNN4 transferred near-miss residual retrieval.
|
| 98 |
-
- `14857698`: rebuild `paper_table_status.*` after near-miss residual summaries.
|
| 99 |
-
|
| 100 |
-
Current scheduler state:
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
near-miss-only residual diagnostics `14857692`-`14857698` are also running or
|
| 105 |
-
dependency-held.
|
| 106 |
|
| 107 |
## Decision Rule For Field Optim Jobs
|
| 108 |
|
|
|
|
| 62 |
|
| 63 |
## Active Jobs
|
| 64 |
|
| 65 |
+
Last checked: `2026-06-28 04:51 UTC`.
|
| 66 |
|
| 67 |
- `14842523`: GPU smoke for `selection_mode=field_optim`.
|
| 68 |
- `14842557`: low-resource CPU unit smoke for the pure action-optimization helper.
|
|
|
|
| 83 |
- `14842619`: KNN4 `retrieval_residual` summary.
|
| 84 |
- `14842646`: CPU unit smoke for the KNN residual helper. Completed.
|
| 85 |
- `14857111`: fixed nearest-1 `retrieval_residual` smoke.
|
| 86 |
+
- `14857112`: fixed nearest-1 `retrieval_residual` full rollout. Completed.
|
| 87 |
+
- `14857113`: fixed nearest-1 `retrieval_residual` summary. Completed.
|
| 88 |
+
- `14857114`: fixed KNN4 `retrieval_residual` smoke. Completed.
|
| 89 |
+
- `14857115`: fixed KNN4 `retrieval_residual` full rollout. Completed.
|
| 90 |
+
- `14857116`: fixed KNN4 `retrieval_residual` summary. Completed.
|
| 91 |
+
- `14857117`: rebuild `paper_table_status.*` after fixed residual summaries. Completed.
|
| 92 |
+
- `14857692`: smoke nearest-1 transferred near-miss residual retrieval. Completed.
|
| 93 |
+
- `14857693`: full nearest-1 transferred near-miss residual retrieval. Canceled.
|
| 94 |
+
- `14857694`: summary nearest-1 transferred near-miss residual retrieval. Canceled.
|
| 95 |
+
- `14857695`: smoke KNN4 transferred near-miss residual retrieval. Completed.
|
| 96 |
+
- `14857696`: full KNN4 transferred near-miss residual retrieval. Canceled.
|
| 97 |
+
- `14857697`: summary KNN4 transferred near-miss residual retrieval. Canceled.
|
| 98 |
+
- `14857698`: rebuild `paper_table_status.*` after near-miss residual summaries. Canceled.
|
| 99 |
+
|
| 100 |
+
Current scheduler state: no tracked jobs are active. `field_optim`,
|
| 101 |
+
`nonexpert_policy_bc5`, and residual v2 jobs completed. Residual nearest-1 is a
|
| 102 |
+
positive clean bridge at 32.12%, KNN4 residual is 29.91%, and near-miss-only
|
| 103 |
+
residual smoke was weak enough to cancel its full jobs.
|
|
|
|
|
|
|
| 104 |
|
| 105 |
## Decision Rule For Field Optim Jobs
|
| 106 |
|
results/paper_table_status.json
CHANGED
|
@@ -158,14 +158,14 @@
|
|
| 158 |
"story_role": "pending transferable local tangent proposal",
|
| 159 |
"fallback_success": null,
|
| 160 |
"pending_job": "14857111/14857112/14857113",
|
| 161 |
-
"path_exists":
|
| 162 |
-
"status": "
|
| 163 |
-
"success":
|
| 164 |
-
"std_success":
|
| 165 |
"completed_seeds": null,
|
| 166 |
-
"num_completed":
|
| 167 |
"best_config": null,
|
| 168 |
-
"gain_vs_h16_policy":
|
| 169 |
},
|
| 170 |
{
|
| 171 |
"key": "retrieval_residual_knn4",
|
|
@@ -177,52 +177,14 @@
|
|
| 177 |
"story_role": "pending KNN tangent proposal",
|
| 178 |
"fallback_success": null,
|
| 179 |
"pending_job": "14857114/14857115/14857116",
|
| 180 |
-
"path_exists":
|
| 181 |
-
"status": "
|
| 182 |
-
"success":
|
| 183 |
-
"std_success":
|
| 184 |
-
"completed_seeds": null,
|
| 185 |
-
"num_completed": null,
|
| 186 |
-
"best_config": null,
|
| 187 |
-
"gain_vs_h16_policy": null
|
| 188 |
-
},
|
| 189 |
-
{
|
| 190 |
-
"key": "retrieval_residual_nearmiss",
|
| 191 |
-
"label": "Train-state near-miss residual retrieval",
|
| 192 |
-
"path": "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_nearmiss_v2_summary.json",
|
| 193 |
-
"clean_deployment": "yes",
|
| 194 |
-
"same_state_proposals": "no",
|
| 195 |
-
"expert_proposal": "no",
|
| 196 |
-
"story_role": "pending transferable near-miss tangent proposal",
|
| 197 |
-
"fallback_success": null,
|
| 198 |
-
"pending_job": "14857692/14857693/14857694",
|
| 199 |
-
"path_exists": false,
|
| 200 |
-
"status": "pending",
|
| 201 |
-
"success": null,
|
| 202 |
-
"std_success": null,
|
| 203 |
-
"completed_seeds": null,
|
| 204 |
-
"num_completed": null,
|
| 205 |
-
"best_config": null,
|
| 206 |
-
"gain_vs_h16_policy": null
|
| 207 |
-
},
|
| 208 |
-
{
|
| 209 |
-
"key": "retrieval_residual_nearmiss_knn4",
|
| 210 |
-
"label": "KNN near-miss residual retrieval",
|
| 211 |
-
"path": "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_nearmiss_knn4_v2_summary.json",
|
| 212 |
-
"clean_deployment": "yes",
|
| 213 |
-
"same_state_proposals": "no",
|
| 214 |
-
"expert_proposal": "no",
|
| 215 |
-
"story_role": "pending KNN near-miss tangent proposal",
|
| 216 |
-
"fallback_success": null,
|
| 217 |
-
"pending_job": "14857695/14857696/14857697",
|
| 218 |
-
"path_exists": false,
|
| 219 |
-
"status": "pending",
|
| 220 |
-
"success": null,
|
| 221 |
-
"std_success": null,
|
| 222 |
"completed_seeds": null,
|
| 223 |
-
"num_completed":
|
| 224 |
"best_config": null,
|
| 225 |
-
"gain_vs_h16_policy":
|
| 226 |
},
|
| 227 |
{
|
| 228 |
"key": "near_miss_only_lattice",
|
|
@@ -349,9 +311,7 @@
|
|
| 349 |
"Do not claim external SOTA from this table alone; add current external baselines separately.",
|
| 350 |
"Current best clean deployment row is Near-miss proposal policy + field at 32.93%.",
|
| 351 |
"Trust-region field optimization should be framed as a negative/diagnostic ablation.",
|
| 352 |
-
"Train-state counterfactual residual retrieval is
|
| 353 |
-
"KNN counterfactual residual retrieval is
|
| 354 |
-
"Train-state near-miss residual retrieval is pending (14857692/14857693/14857694).",
|
| 355 |
-
"KNN near-miss residual retrieval is pending (14857695/14857696/14857697)."
|
| 356 |
]
|
| 357 |
}
|
|
|
|
| 158 |
"story_role": "pending transferable local tangent proposal",
|
| 159 |
"fallback_success": null,
|
| 160 |
"pending_job": "14857111/14857112/14857113",
|
| 161 |
+
"path_exists": true,
|
| 162 |
+
"status": "complete",
|
| 163 |
+
"success": 0.32115942028985506,
|
| 164 |
+
"std_success": 0.012581179370556922,
|
| 165 |
"completed_seeds": null,
|
| 166 |
+
"num_completed": 3,
|
| 167 |
"best_config": null,
|
| 168 |
+
"gain_vs_h16_policy": 0.02376811594202899
|
| 169 |
},
|
| 170 |
{
|
| 171 |
"key": "retrieval_residual_knn4",
|
|
|
|
| 177 |
"story_role": "pending KNN tangent proposal",
|
| 178 |
"fallback_success": null,
|
| 179 |
"pending_job": "14857114/14857115/14857116",
|
| 180 |
+
"path_exists": true,
|
| 181 |
+
"status": "complete",
|
| 182 |
+
"success": 0.2991304347826087,
|
| 183 |
+
"std_success": 0.02005663059942746,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
"completed_seeds": null,
|
| 185 |
+
"num_completed": 3,
|
| 186 |
"best_config": null,
|
| 187 |
+
"gain_vs_h16_policy": 0.001739130434782632
|
| 188 |
},
|
| 189 |
{
|
| 190 |
"key": "near_miss_only_lattice",
|
|
|
|
| 311 |
"Do not claim external SOTA from this table alone; add current external baselines separately.",
|
| 312 |
"Current best clean deployment row is Near-miss proposal policy + field at 32.93%.",
|
| 313 |
"Trust-region field optimization should be framed as a negative/diagnostic ablation.",
|
| 314 |
+
"Train-state counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.",
|
| 315 |
+
"KNN counterfactual residual retrieval is a positive clean bridge but remains below the current clean best."
|
|
|
|
|
|
|
| 316 |
]
|
| 317 |
}
|
results/paper_table_status.md
CHANGED
|
@@ -11,10 +11,8 @@ Baseline h=16 policy: 29.74%
|
|
| 11 |
| field_optim | Trust-region field optimization | complete k32_sigma0.50 | 25.39% | -4.35 pp | yes | no | no | differentiable field-ascent diagnostic |
|
| 12 |
| nonexpert_policy_bc5 | Best non-expert proposal policy | complete | 27.88% | -1.86 pp | yes | no | no | pending broader non-expert proposal model |
|
| 13 |
| nonexpert_policy_bc5_field | Best non-expert proposal policy + field | complete k64_sigma0.50 | 26.49% | -3.25 pp | yes | no | no | pending broader proposal-field bridge |
|
| 14 |
-
| retrieval_residual | Train-state counterfactual residual retrieval |
|
| 15 |
-
| retrieval_residual_knn4 | KNN counterfactual residual retrieval |
|
| 16 |
-
| retrieval_residual_nearmiss | Train-state near-miss residual retrieval | pending 14857692/14857693/14857694 | pending | pending | yes | no | no | pending transferable near-miss tangent proposal |
|
| 17 |
-
| retrieval_residual_nearmiss_knn4 | KNN near-miss residual retrieval | pending 14857695/14857696/14857697 | pending | pending | yes | no | no | pending KNN near-miss tangent proposal |
|
| 18 |
| near_miss_only_lattice | Same-state lattice, near-miss only | complete | 55.94% | +26.20 pp | no | yes | no | minimal mechanism result |
|
| 19 |
| no_expert_lattice | Same-state lattice, no expert | complete | 56.99% | +27.25 pp | no | yes | no | main conservative mechanism result |
|
| 20 |
| no_near_miss_no_expert_lattice | Same-state lattice, no expert/no near-miss | complete | 25.57% | -4.17 pp | no | yes | no | mechanism knockout |
|
|
@@ -27,7 +25,5 @@ Baseline h=16 policy: 29.74%
|
|
| 27 |
- Do not claim external SOTA from this table alone; add current external baselines separately.
|
| 28 |
- Current best clean deployment row is Near-miss proposal policy + field at 32.93%.
|
| 29 |
- Trust-region field optimization should be framed as a negative/diagnostic ablation.
|
| 30 |
-
- Train-state counterfactual residual retrieval is
|
| 31 |
-
- KNN counterfactual residual retrieval is
|
| 32 |
-
- Train-state near-miss residual retrieval is pending (14857692/14857693/14857694).
|
| 33 |
-
- KNN near-miss residual retrieval is pending (14857695/14857696/14857697).
|
|
|
|
| 11 |
| field_optim | Trust-region field optimization | complete k32_sigma0.50 | 25.39% | -4.35 pp | yes | no | no | differentiable field-ascent diagnostic |
|
| 12 |
| nonexpert_policy_bc5 | Best non-expert proposal policy | complete | 27.88% | -1.86 pp | yes | no | no | pending broader non-expert proposal model |
|
| 13 |
| nonexpert_policy_bc5_field | Best non-expert proposal policy + field | complete k64_sigma0.50 | 26.49% | -3.25 pp | yes | no | no | pending broader proposal-field bridge |
|
| 14 |
+
| retrieval_residual | Train-state counterfactual residual retrieval | complete | 32.12% | +2.38 pp | yes | no | no | pending transferable local tangent proposal |
|
| 15 |
+
| retrieval_residual_knn4 | KNN counterfactual residual retrieval | complete | 29.91% | +0.17 pp | yes | no | no | pending KNN tangent proposal |
|
|
|
|
|
|
|
| 16 |
| near_miss_only_lattice | Same-state lattice, near-miss only | complete | 55.94% | +26.20 pp | no | yes | no | minimal mechanism result |
|
| 17 |
| no_expert_lattice | Same-state lattice, no expert | complete | 56.99% | +27.25 pp | no | yes | no | main conservative mechanism result |
|
| 18 |
| no_near_miss_no_expert_lattice | Same-state lattice, no expert/no near-miss | complete | 25.57% | -4.17 pp | no | yes | no | mechanism knockout |
|
|
|
|
| 25 |
- Do not claim external SOTA from this table alone; add current external baselines separately.
|
| 26 |
- Current best clean deployment row is Near-miss proposal policy + field at 32.93%.
|
| 27 |
- Trust-region field optimization should be framed as a negative/diagnostic ablation.
|
| 28 |
+
- Train-state counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.
|
| 29 |
+
- KNN counterfactual residual retrieval is a positive clean bridge but remains below the current clean best.
|
|
|
|
|
|
scripts/build_paper_table_status.py
CHANGED
|
@@ -96,44 +96,44 @@ SPECS = [
|
|
| 96 |
pending_job="14842574/14842577/14842617",
|
| 97 |
),
|
| 98 |
ResultSpec(
|
| 99 |
-
key="
|
| 100 |
-
label="
|
| 101 |
-
path="
|
| 102 |
clean_deployment="yes",
|
| 103 |
same_state_proposals="no",
|
| 104 |
expert_proposal="no",
|
| 105 |
-
story_role="pending
|
| 106 |
-
pending_job="
|
| 107 |
),
|
| 108 |
ResultSpec(
|
| 109 |
-
key="
|
| 110 |
-
label="
|
| 111 |
-
path="
|
| 112 |
clean_deployment="yes",
|
| 113 |
same_state_proposals="no",
|
| 114 |
expert_proposal="no",
|
| 115 |
-
story_role="pending
|
| 116 |
-
pending_job="
|
| 117 |
),
|
| 118 |
ResultSpec(
|
| 119 |
-
key="
|
| 120 |
-
label="Train-state
|
| 121 |
-
path="
|
| 122 |
clean_deployment="yes",
|
| 123 |
same_state_proposals="no",
|
| 124 |
expert_proposal="no",
|
| 125 |
-
story_role="pending transferable
|
| 126 |
-
pending_job="
|
| 127 |
),
|
| 128 |
ResultSpec(
|
| 129 |
-
key="
|
| 130 |
-
label="KNN
|
| 131 |
-
path="
|
| 132 |
clean_deployment="yes",
|
| 133 |
same_state_proposals="no",
|
| 134 |
expert_proposal="no",
|
| 135 |
-
story_role="pending KNN
|
| 136 |
-
pending_job="
|
| 137 |
),
|
| 138 |
ResultSpec(
|
| 139 |
key="near_miss_only_lattice",
|
|
@@ -287,8 +287,6 @@ def _decision_notes(rows: list[dict[str, Any]]) -> list[str]:
|
|
| 287 |
"field_optim",
|
| 288 |
"retrieval_residual",
|
| 289 |
"retrieval_residual_knn4",
|
| 290 |
-
"retrieval_residual_nearmiss",
|
| 291 |
-
"retrieval_residual_nearmiss_knn4",
|
| 292 |
):
|
| 293 |
row = by_key[key]
|
| 294 |
if row["success"] is None:
|
|
@@ -297,6 +295,10 @@ def _decision_notes(rows: list[dict[str, Any]]) -> list[str]:
|
|
| 297 |
notes.append(f"{row['label']} is strong enough to promote as a clean bridge.")
|
| 298 |
elif row["success"] > 0.3293:
|
| 299 |
notes.append(f"{row['label']} improves the clean bridge but is not yet the main result.")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 300 |
else:
|
| 301 |
notes.append(f"{row['label']} should be framed as a negative/diagnostic ablation.")
|
| 302 |
return notes
|
|
|
|
| 96 |
pending_job="14842574/14842577/14842617",
|
| 97 |
),
|
| 98 |
ResultSpec(
|
| 99 |
+
key="field_selected_noexpert_policy",
|
| 100 |
+
label="Field-selected no-expert distillation policy",
|
| 101 |
+
path="h16_policy_ckpt_field_selected_noexpert_bc5_summary.json",
|
| 102 |
clean_deployment="yes",
|
| 103 |
same_state_proposals="no",
|
| 104 |
expert_proposal="no",
|
| 105 |
+
story_role="pending student of field-on-lattice teacher",
|
| 106 |
+
pending_job="14858327/14858328/14858329/14858330",
|
| 107 |
),
|
| 108 |
ResultSpec(
|
| 109 |
+
key="field_selected_noexpert_policy_field",
|
| 110 |
+
label="Field-selected no-expert distillation + field",
|
| 111 |
+
path="h16_policy_ckpt_field_selected_noexpert_bc5_bestpt_field_sweep_summary.json",
|
| 112 |
clean_deployment="yes",
|
| 113 |
same_state_proposals="no",
|
| 114 |
expert_proposal="no",
|
| 115 |
+
story_role="pending student proposal with field scoring",
|
| 116 |
+
pending_job="14858327/14858328/14858331/14858332",
|
| 117 |
),
|
| 118 |
ResultSpec(
|
| 119 |
+
key="retrieval_residual",
|
| 120 |
+
label="Train-state counterfactual residual retrieval",
|
| 121 |
+
path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_v2_summary.json",
|
| 122 |
clean_deployment="yes",
|
| 123 |
same_state_proposals="no",
|
| 124 |
expert_proposal="no",
|
| 125 |
+
story_role="pending transferable local tangent proposal",
|
| 126 |
+
pending_job="14857111/14857112/14857113",
|
| 127 |
),
|
| 128 |
ResultSpec(
|
| 129 |
+
key="retrieval_residual_knn4",
|
| 130 |
+
label="KNN counterfactual residual retrieval",
|
| 131 |
+
path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn4_v2_summary.json",
|
| 132 |
clean_deployment="yes",
|
| 133 |
same_state_proposals="no",
|
| 134 |
expert_proposal="no",
|
| 135 |
+
story_role="pending KNN tangent proposal",
|
| 136 |
+
pending_job="14857114/14857115/14857116",
|
| 137 |
),
|
| 138 |
ResultSpec(
|
| 139 |
key="near_miss_only_lattice",
|
|
|
|
| 287 |
"field_optim",
|
| 288 |
"retrieval_residual",
|
| 289 |
"retrieval_residual_knn4",
|
|
|
|
|
|
|
| 290 |
):
|
| 291 |
row = by_key[key]
|
| 292 |
if row["success"] is None:
|
|
|
|
| 295 |
notes.append(f"{row['label']} is strong enough to promote as a clean bridge.")
|
| 296 |
elif row["success"] > 0.3293:
|
| 297 |
notes.append(f"{row['label']} improves the clean bridge but is not yet the main result.")
|
| 298 |
+
elif row["success"] > BASELINE_H16_POLICY:
|
| 299 |
+
notes.append(
|
| 300 |
+
f"{row['label']} is a positive clean bridge but remains below the current clean best."
|
| 301 |
+
)
|
| 302 |
else:
|
| 303 |
notes.append(f"{row['label']} should be framed as a negative/diagnostic ablation.")
|
| 304 |
return notes
|
scripts/export_field_selected_policy_targets.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import json
|
| 6 |
+
import sys
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Any
|
| 9 |
+
|
| 10 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 11 |
+
if str(PROJECT_ROOT) not in sys.path:
|
| 12 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 13 |
+
|
| 14 |
+
from dovla_cil.data.datasets import CILDataset # noqa: E402
|
| 15 |
+
from dovla_cil.eval.lattice_eval import _validation_group_ids # noqa: E402
|
| 16 |
+
from dovla_cil.eval.maniskill_policy_rollout import _numeric_action_values # noqa: E402
|
| 17 |
+
from dovla_cil.models.dovla import ( # noqa: E402
|
| 18 |
+
DoVLAConfig,
|
| 19 |
+
DoVLAModel,
|
| 20 |
+
load_model_state,
|
| 21 |
+
vectorize_toy_observation,
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def main(argv: list[str] | None = None) -> int:
|
| 26 |
+
parser = argparse.ArgumentParser(
|
| 27 |
+
description="Export policy BC targets chosen by a trained field on CIL action lattices."
|
| 28 |
+
)
|
| 29 |
+
parser.add_argument("--checkpoint", type=Path, required=True)
|
| 30 |
+
parser.add_argument("--dataset", type=Path, required=True)
|
| 31 |
+
parser.add_argument("--out", type=Path, required=True)
|
| 32 |
+
parser.add_argument("--device", default="auto")
|
| 33 |
+
parser.add_argument("--split", choices=("train", "val", "all"), default="train")
|
| 34 |
+
parser.add_argument("--batch-groups", type=int, default=32)
|
| 35 |
+
parser.add_argument(
|
| 36 |
+
"--exclude-types",
|
| 37 |
+
default="expert",
|
| 38 |
+
help="Comma-separated candidate_type values to exclude before field selection.",
|
| 39 |
+
)
|
| 40 |
+
parser.add_argument("--max-groups", type=int, default=None)
|
| 41 |
+
args = parser.parse_args(argv)
|
| 42 |
+
|
| 43 |
+
if args.batch_groups <= 0:
|
| 44 |
+
parser.error("--batch-groups must be positive")
|
| 45 |
+
if args.max_groups is not None and args.max_groups <= 0:
|
| 46 |
+
parser.error("--max-groups must be positive when provided")
|
| 47 |
+
|
| 48 |
+
try:
|
| 49 |
+
import torch
|
| 50 |
+
except ImportError as exc: # pragma: no cover
|
| 51 |
+
raise ImportError("export_field_selected_policy_targets.py requires torch") from exc
|
| 52 |
+
|
| 53 |
+
checkpoint = torch.load(
|
| 54 |
+
args.checkpoint,
|
| 55 |
+
map_location=_resolve_device(args.device),
|
| 56 |
+
weights_only=False,
|
| 57 |
+
)
|
| 58 |
+
model_config = DoVLAConfig(**checkpoint["model_config"])
|
| 59 |
+
if model_config.observation_mode != "state":
|
| 60 |
+
raise ValueError("field-selected target export currently supports state observations only")
|
| 61 |
+
device = _resolve_device(args.device)
|
| 62 |
+
model = DoVLAModel(model_config).to(device)
|
| 63 |
+
load_model_state(model, checkpoint)
|
| 64 |
+
model.eval()
|
| 65 |
+
|
| 66 |
+
dataset = CILDataset(args.dataset)
|
| 67 |
+
trainer_config = checkpoint.get("trainer_config", {})
|
| 68 |
+
val_ids = set(
|
| 69 |
+
_validation_group_ids(
|
| 70 |
+
dataset.group_ids,
|
| 71 |
+
val_fraction=float(trainer_config.get("val_fraction", 0.2)),
|
| 72 |
+
seed=int(trainer_config.get("seed", 0)),
|
| 73 |
+
)
|
| 74 |
+
)
|
| 75 |
+
if args.split == "train":
|
| 76 |
+
group_ids = [group_id for group_id in dataset.group_ids if group_id not in val_ids]
|
| 77 |
+
elif args.split == "val":
|
| 78 |
+
group_ids = [group_id for group_id in dataset.group_ids if group_id in val_ids]
|
| 79 |
+
else:
|
| 80 |
+
group_ids = list(dataset.group_ids)
|
| 81 |
+
if args.max_groups is not None:
|
| 82 |
+
group_ids = group_ids[: args.max_groups]
|
| 83 |
+
|
| 84 |
+
excluded = {item.strip() for item in args.exclude_types.split(",") if item.strip()}
|
| 85 |
+
targets: dict[str, dict[str, Any]] = {}
|
| 86 |
+
counts: dict[str, int] = {}
|
| 87 |
+
with torch.no_grad():
|
| 88 |
+
for start in range(0, len(group_ids), args.batch_groups):
|
| 89 |
+
for group_id in group_ids[start : start + args.batch_groups]:
|
| 90 |
+
records = [
|
| 91 |
+
record
|
| 92 |
+
for record in dataset.get_group(group_id)
|
| 93 |
+
if record.candidate_type not in excluded
|
| 94 |
+
]
|
| 95 |
+
if not records:
|
| 96 |
+
records = dataset.get_group(group_id)
|
| 97 |
+
if not records:
|
| 98 |
+
continue
|
| 99 |
+
obs = torch.tensor(
|
| 100 |
+
[
|
| 101 |
+
vectorize_toy_observation(
|
| 102 |
+
records[0].observation_inline or {},
|
| 103 |
+
obs_dim=model_config.obs_dim,
|
| 104 |
+
)
|
| 105 |
+
]
|
| 106 |
+
* len(records),
|
| 107 |
+
dtype=torch.float32,
|
| 108 |
+
device=device,
|
| 109 |
+
)
|
| 110 |
+
actions = torch.tensor(
|
| 111 |
+
[_numeric_action_values(record) for record in records],
|
| 112 |
+
dtype=torch.float32,
|
| 113 |
+
device=device,
|
| 114 |
+
)
|
| 115 |
+
field = model.forward_field(
|
| 116 |
+
obs,
|
| 117 |
+
[record.instruction for record in records],
|
| 118 |
+
actions,
|
| 119 |
+
)
|
| 120 |
+
best_idx = int(torch.argmax(field["potential"].reshape(len(records))).item())
|
| 121 |
+
best = records[best_idx]
|
| 122 |
+
counts[best.candidate_type] = counts.get(best.candidate_type, 0) + 1
|
| 123 |
+
targets[group_id] = {
|
| 124 |
+
"record_id": best.record_id,
|
| 125 |
+
"candidate_type": best.candidate_type,
|
| 126 |
+
"task_id": best.task_id,
|
| 127 |
+
"score": float(best.reward.score),
|
| 128 |
+
"rank_within_group": best.rank_within_group,
|
| 129 |
+
}
|
| 130 |
+
|
| 131 |
+
payload = {
|
| 132 |
+
"checkpoint": str(args.checkpoint),
|
| 133 |
+
"dataset": str(args.dataset),
|
| 134 |
+
"split": args.split,
|
| 135 |
+
"excluded_candidate_types": sorted(excluded),
|
| 136 |
+
"num_groups": len(group_ids),
|
| 137 |
+
"num_targets": len(targets),
|
| 138 |
+
"selected_candidate_type_counts": counts,
|
| 139 |
+
"targets": targets,
|
| 140 |
+
}
|
| 141 |
+
args.out.parent.mkdir(parents=True, exist_ok=True)
|
| 142 |
+
args.out.write_text(json.dumps(payload, indent=2) + "\n")
|
| 143 |
+
print(json.dumps({k: v for k, v in payload.items() if k != "targets"}, indent=2))
|
| 144 |
+
print(f"Wrote {args.out}")
|
| 145 |
+
return 0
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def _resolve_device(device: str) -> str:
|
| 149 |
+
if device != "auto":
|
| 150 |
+
return device
|
| 151 |
+
try:
|
| 152 |
+
import torch
|
| 153 |
+
except ImportError: # pragma: no cover
|
| 154 |
+
return "cpu"
|
| 155 |
+
return "cuda" if torch.cuda.is_available() else "cpu"
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
if __name__ == "__main__":
|
| 159 |
+
raise SystemExit(main())
|
scripts/slurm/train_dovla_h16_policy_ckpt.sbatch
CHANGED
|
@@ -26,6 +26,7 @@ DATASET="${DATASET:-$SCRATCH_ROOT/experiments/h16_merged_dataset}"
|
|
| 26 |
RUN_ROOT="${RUN_ROOT:-$SCRATCH_ROOT/experiments/dovla_h16_policy_ckpt_runs}"
|
| 27 |
OBJECTIVE="${OBJECTIVE:-base}"
|
| 28 |
POLICY_TARGET_TYPES="${POLICY_TARGET_TYPES:-}"
|
|
|
|
| 29 |
SEED=$SLURM_ARRAY_TASK_ID
|
| 30 |
OUT_DIR="$RUN_ROOT/$OBJECTIVE/seed_$SEED"
|
| 31 |
|
|
@@ -50,6 +51,9 @@ TRAIN_EXTRA_ARGS=()
|
|
| 50 |
if [[ -n "$POLICY_TARGET_TYPES" ]]; then
|
| 51 |
TRAIN_EXTRA_ARGS+=(--policy-target-types "$POLICY_TARGET_TYPES")
|
| 52 |
fi
|
|
|
|
|
|
|
|
|
|
| 53 |
if [[ -n "${EXTRA_TRAIN_ARGS:-}" ]]; then
|
| 54 |
# shellcheck disable=SC2206
|
| 55 |
EXTRA_SPLIT=($EXTRA_TRAIN_ARGS)
|
|
@@ -62,6 +66,7 @@ echo "Seed: $SEED"
|
|
| 62 |
echo "Dataset: $DATASET"
|
| 63 |
echo "Output: $OUT_DIR"
|
| 64 |
echo "Policy target types: ${POLICY_TARGET_TYPES:-<best-any>}"
|
|
|
|
| 65 |
echo "=================================================="
|
| 66 |
|
| 67 |
"${PYTHON_CMD[@]}" -c "
|
|
|
|
| 26 |
RUN_ROOT="${RUN_ROOT:-$SCRATCH_ROOT/experiments/dovla_h16_policy_ckpt_runs}"
|
| 27 |
OBJECTIVE="${OBJECTIVE:-base}"
|
| 28 |
POLICY_TARGET_TYPES="${POLICY_TARGET_TYPES:-}"
|
| 29 |
+
POLICY_TARGET_MAP="${POLICY_TARGET_MAP:-}"
|
| 30 |
SEED=$SLURM_ARRAY_TASK_ID
|
| 31 |
OUT_DIR="$RUN_ROOT/$OBJECTIVE/seed_$SEED"
|
| 32 |
|
|
|
|
| 51 |
if [[ -n "$POLICY_TARGET_TYPES" ]]; then
|
| 52 |
TRAIN_EXTRA_ARGS+=(--policy-target-types "$POLICY_TARGET_TYPES")
|
| 53 |
fi
|
| 54 |
+
if [[ -n "$POLICY_TARGET_MAP" ]]; then
|
| 55 |
+
TRAIN_EXTRA_ARGS+=(--policy-target-map "$POLICY_TARGET_MAP")
|
| 56 |
+
fi
|
| 57 |
if [[ -n "${EXTRA_TRAIN_ARGS:-}" ]]; then
|
| 58 |
# shellcheck disable=SC2206
|
| 59 |
EXTRA_SPLIT=($EXTRA_TRAIN_ARGS)
|
|
|
|
| 66 |
echo "Dataset: $DATASET"
|
| 67 |
echo "Output: $OUT_DIR"
|
| 68 |
echo "Policy target types: ${POLICY_TARGET_TYPES:-<best-any>}"
|
| 69 |
+
echo "Policy target map: ${POLICY_TARGET_MAP:-<none>}"
|
| 70 |
echo "=================================================="
|
| 71 |
|
| 72 |
"${PYTHON_CMD[@]}" -c "
|
scripts/train_dovla.py
CHANGED
|
@@ -85,6 +85,13 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 85 |
help="Comma-separated candidate_type filter for policy BC targets. "
|
| 86 |
"Empty means best action from every group.",
|
| 87 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
parser.add_argument(
|
| 89 |
"--loss-weight",
|
| 90 |
action="append",
|
|
@@ -129,6 +136,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 129 |
policy_target_types=tuple(
|
| 130 |
item.strip() for item in args.policy_target_types.split(",") if item.strip()
|
| 131 |
),
|
|
|
|
| 132 |
losses=loss_weights,
|
| 133 |
)
|
| 134 |
result = DoVLATrainer(config).train()
|
|
|
|
| 85 |
help="Comma-separated candidate_type filter for policy BC targets. "
|
| 86 |
"Empty means best action from every group.",
|
| 87 |
)
|
| 88 |
+
parser.add_argument(
|
| 89 |
+
"--policy-target-map",
|
| 90 |
+
type=Path,
|
| 91 |
+
default=None,
|
| 92 |
+
help="JSON mapping group_id to policy BC target record_id. Missing groups fall back "
|
| 93 |
+
"to --policy-target-types or best-in-group.",
|
| 94 |
+
)
|
| 95 |
parser.add_argument(
|
| 96 |
"--loss-weight",
|
| 97 |
action="append",
|
|
|
|
| 136 |
policy_target_types=tuple(
|
| 137 |
item.strip() for item in args.policy_target_types.split(",") if item.strip()
|
| 138 |
),
|
| 139 |
+
policy_target_map=args.policy_target_map,
|
| 140 |
losses=loss_weights,
|
| 141 |
)
|
| 142 |
result = DoVLATrainer(config).train()
|
tests/test_trainer.py
CHANGED
|
@@ -141,3 +141,40 @@ def test_policy_target_type_filter_selects_best_allowed_candidate() -> None:
|
|
| 141 |
"g0": "near",
|
| 142 |
"g1": "fallback",
|
| 143 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
"g0": "near",
|
| 142 |
"g1": "fallback",
|
| 143 |
}
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def test_policy_target_map_overrides_group_target_with_fallback() -> None:
|
| 147 |
+
records = [
|
| 148 |
+
SimpleNamespace(
|
| 149 |
+
group_id="g0",
|
| 150 |
+
candidate_type="expert",
|
| 151 |
+
reward=SimpleNamespace(score=2.0),
|
| 152 |
+
rank_within_group=0,
|
| 153 |
+
record_id="expert",
|
| 154 |
+
),
|
| 155 |
+
SimpleNamespace(
|
| 156 |
+
group_id="g0",
|
| 157 |
+
candidate_type="near_miss",
|
| 158 |
+
reward=SimpleNamespace(score=1.5),
|
| 159 |
+
rank_within_group=1,
|
| 160 |
+
record_id="field_choice",
|
| 161 |
+
),
|
| 162 |
+
SimpleNamespace(
|
| 163 |
+
group_id="g1",
|
| 164 |
+
candidate_type="near_miss",
|
| 165 |
+
reward=SimpleNamespace(score=0.7),
|
| 166 |
+
rank_within_group=1,
|
| 167 |
+
record_id="fallback",
|
| 168 |
+
),
|
| 169 |
+
]
|
| 170 |
+
|
| 171 |
+
selected = _best_records_by_group(
|
| 172 |
+
records,
|
| 173 |
+
candidate_types=("near_miss",),
|
| 174 |
+
target_record_ids={"g0": "field_choice"},
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
assert {record.group_id: record.record_id for record in selected} == {
|
| 178 |
+
"g0": "field_choice",
|
| 179 |
+
"g1": "fallback",
|
| 180 |
+
}
|