Auto-sync: 2026-06-29 12:01:54 (part 4)
Browse files- results/paper_analysis.json +6 -1
- results/paper_analysis.md +2 -1
- results/paper_core_results.md +37 -31
- results/paper_story_memo.md +77 -57
- results/paper_table_status.json +19 -0
- results/paper_table_status.md +1 -0
- scripts/build_paper_analysis.py +8 -0
- scripts/build_paper_table_status.py +10 -0
- scripts/eval_maniskill_policy_rollout.py +7 -0
- scripts/slurm/eval_maniskill_policy_rollout.sbatch +4 -0
- scripts/slurm/eval_maniskill_policy_rollout_cpu_smoke.sbatch +4 -0
- scripts/slurm/summarize_h16_policy_ckpt.sbatch +3 -0
- tests/test_maniskill_policy_rollout.py +55 -0
results/paper_analysis.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"best_clean_key": "residual_k4_composemasked_grid035040045_noopbonus003",
|
| 3 |
-
"generated_utc": "2026-06-
|
| 4 |
"mechanism_gap": {
|
| 5 |
"best_clean_vs_direct_same_ckpt": 0.07246376811594196,
|
| 6 |
"best_clean_vs_h16": 0.0579710144927536,
|
|
@@ -844,6 +844,11 @@
|
|
| 844 |
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_compose_grid035040045_safe_margin0p20_noopbonus0p03_summary.json",
|
| 845 |
"std_success": 0.01578047256674342
|
| 846 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 847 |
"residual_k4_composemasked_grid035040045": {
|
| 848 |
"ci95_success": 0.03024411731871433,
|
| 849 |
"label": "K4 composed type-consensus tangents, masked, scales 0.35/0.40/0.45",
|
|
|
|
| 1 |
{
|
| 2 |
"best_clean_key": "residual_k4_composemasked_grid035040045_noopbonus003",
|
| 3 |
+
"generated_utc": "2026-06-29T16:13:16+00:00",
|
| 4 |
"mechanism_gap": {
|
| 5 |
"best_clean_vs_direct_same_ckpt": 0.07246376811594196,
|
| 6 |
"best_clean_vs_h16": 0.0579710144927536,
|
|
|
|
| 844 |
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_compose_grid035040045_safe_margin0p20_noopbonus0p03_summary.json",
|
| 845 |
"std_success": 0.01578047256674342
|
| 846 |
},
|
| 847 |
+
"residual_k4_composemasked_compbonus_grid035040045_noopbonus003": {
|
| 848 |
+
"label": "K4 composed type-consensus tangents, masked, component no-op bonus 0.03",
|
| 849 |
+
"missing": true,
|
| 850 |
+
"source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_compbonus_grid035040045_safe_margin0p20_noopbonus0p03_summary.json"
|
| 851 |
+
},
|
| 852 |
"residual_k4_composemasked_grid035040045": {
|
| 853 |
"ci95_success": 0.03024411731871433,
|
| 854 |
"label": "K4 composed type-consensus tangents, masked, scales 0.35/0.40/0.45",
|
results/paper_analysis.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
# Paper Analysis
|
| 2 |
|
| 3 |
-
Generated: `2026-06-
|
| 4 |
|
| 5 |
## Main Seed Statistics
|
| 6 |
|
|
@@ -47,6 +47,7 @@ Generated: `2026-06-29T15:56:21+00:00`
|
|
| 47 |
| residual_k4_compose_grid035040045_noopbonus003 | K4 composed type-consensus tangents, scales 0.35/0.40/0.45, no-op bonus 0.03 | 3 | 34.14% +/- 1.58 | +/- 3.92 | 56.00% | 0.482 | +4.41 pp |
|
| 48 |
| residual_k4_composemasked_grid035040045 | K4 composed type-consensus tangents, masked, scales 0.35/0.40/0.45 | 3 | 35.30% +/- 1.22 | +/- 3.02 | 56.91% | 0.410 | +5.57 pp |
|
| 49 |
| residual_k4_composemasked_grid035040045_noopbonus003 | K4 composed type-consensus tangents, masked, scales 0.35/0.40/0.45, no-op bonus 0.03 | 3 | 35.54% +/- 1.02 | +/- 2.53 | 57.02% | 0.411 | +5.80 pp |
|
|
|
|
| 50 |
| repair_nearmiss_k4_grid025035050_margin020 | K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20 | 3 | 34.32% +/- 1.35 | +/- 3.36 | 55.97% | 0.394 | +4.58 pp |
|
| 51 |
| repair_nearmiss_k4_grid035050075_margin020 | K4 near-miss-to-expert repair tangent, scales 0.35/0.50/0.75, margin 0.20 | 3 | 34.38% +/- 1.50 | +/- 3.73 | 56.05% | 0.394 | +4.64 pp |
|
| 52 |
| repair_nearmiss_k4_grid025035050_margin010 | K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.10 | 3 | 34.14% +/- 1.48 | +/- 3.67 | 56.01% | 0.393 | +4.41 pp |
|
|
|
|
| 1 |
# Paper Analysis
|
| 2 |
|
| 3 |
+
Generated: `2026-06-29T16:13:16+00:00`
|
| 4 |
|
| 5 |
## Main Seed Statistics
|
| 6 |
|
|
|
|
| 47 |
| residual_k4_compose_grid035040045_noopbonus003 | K4 composed type-consensus tangents, scales 0.35/0.40/0.45, no-op bonus 0.03 | 3 | 34.14% +/- 1.58 | +/- 3.92 | 56.00% | 0.482 | +4.41 pp |
|
| 48 |
| residual_k4_composemasked_grid035040045 | K4 composed type-consensus tangents, masked, scales 0.35/0.40/0.45 | 3 | 35.30% +/- 1.22 | +/- 3.02 | 56.91% | 0.410 | +5.57 pp |
|
| 49 |
| residual_k4_composemasked_grid035040045_noopbonus003 | K4 composed type-consensus tangents, masked, scales 0.35/0.40/0.45, no-op bonus 0.03 | 3 | 35.54% +/- 1.02 | +/- 2.53 | 57.02% | 0.411 | +5.80 pp |
|
| 50 |
+
| residual_k4_composemasked_compbonus_grid035040045_noopbonus003 | K4 composed type-consensus tangents, masked, component no-op bonus 0.03 | 0 | missing | missing | missing | missing | missing |
|
| 51 |
| repair_nearmiss_k4_grid025035050_margin020 | K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20 | 3 | 34.32% +/- 1.35 | +/- 3.36 | 55.97% | 0.394 | +4.58 pp |
|
| 52 |
| repair_nearmiss_k4_grid035050075_margin020 | K4 near-miss-to-expert repair tangent, scales 0.35/0.50/0.75, margin 0.20 | 3 | 34.38% +/- 1.50 | +/- 3.73 | 56.05% | 0.394 | +4.64 pp |
|
| 53 |
| repair_nearmiss_k4_grid025035050_margin010 | K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.10 | 3 | 34.14% +/- 1.48 | +/- 3.67 | 56.01% | 0.393 | +4.41 pp |
|
results/paper_core_results.md
CHANGED
|
@@ -5,9 +5,10 @@ baseline is the h=16 rank-checkpoint online rollout (`29.74%`).
|
|
| 5 |
|
| 6 |
For paired seed deltas, per-task gaps, and selection histograms, regenerate and
|
| 7 |
read `paper_analysis.md` with `python3 scripts/build_paper_analysis.py`. Current
|
| 8 |
-
paired analysis: best clean K4
|
| 9 |
-
no-op prior is `+5.
|
| 10 |
-
`+27.25 pp`, and the remaining clean-to-same-state proposal gap is
|
|
|
|
| 11 |
|
| 12 |
| Method | Uses same-state proposals | Uses expert proposal | Success | Gain vs policy | Interpretation |
|
| 13 |
|---|---:|---:|---:|---:|---|
|
|
@@ -41,17 +42,19 @@ no-op prior is `+5.68 pp` over canonical h=16, same-state no-expert lattice is
|
|
| 41 |
| K2 task-relative residual retrieval, safe residuals + advantage margin 0.20 | No | No | 34.26% | +4.52 pp | Actor-pose-only retrieval is too lossy; raw full-state similarity is better for residual transfer |
|
| 42 |
| K4 train-state residual retrieval, safe residuals + mean-by-type tangent consensus | No | No | 34.96% | +5.22 pp | Near-tie clean diagnostic; consensus alone does not beat raw K2 residuals |
|
| 43 |
| K4 mean-by-type residual retrieval + no-op prior 0.03 | No | No | 35.25% | +5.51 pp | Previous fixed-scale clean plateau; 0.025-0.035 nudges high-value no-op residuals without changing the core proposal family |
|
| 44 |
-
| K4 mean-by-type residual retrieval + scale-grid no-op prior 0.03 | No | No | 35.42% | +5.68 pp |
|
| 45 |
| K4 mean-by-type residual retrieval + source-progress prior 0.03 | No | No | 35.25% | +5.51 pp | Ties the fixed-scale typed prior without a hand typed no-op prior; train-measured source progress can replace but not improve the typed prior |
|
| 46 |
| K4 mean-by-type residual retrieval + source-progress prior 0.05 | No | No | 35.13% | +5.39 pp | A stronger measured-progress prior over-selects nonzero residuals and drops below the plateau |
|
| 47 |
| K4 mean-by-type residual retrieval + source-score prior 0.015/0.020 | No | No | 35.25% | +5.51 pp | Full train reward score, including terminal success, also replaces the fixed-scale typed prior without improving it |
|
| 48 |
| K4 mean-by-type residual retrieval + scale-grid source-score prior 0.02 | No | No | 35.30% | +5.57 pp | Measured train-source prior benefits from tangent length calibration but remains below the typed no-op scale-grid row |
|
| 49 |
| K4 mean-by-type residual retrieval + upper scale-grid no-op prior 0.03 | No | No | 35.36% | +5.62 pp | Scales 0.40/0.45/0.50 nearly tie but do not beat the 0.35/0.40/0.45 row |
|
| 50 |
| K4 mean-by-type residual retrieval + wide scale-grid no-op prior 0.03 | No | No | 35.13% | +5.39 pp | Including 0.55 over-extends the transported tangent and drops below the best local calibration |
|
| 51 |
-
| K4 mean-by-type residual retrieval + minimum-energy action penalty | No | No | 35.36-35.42% | +5.62-5.68 pp | A tiny action L2 penalty (0.05) ties the
|
| 52 |
| K4 mean-by-type residual retrieval + source-advantage prior/gate | No | No | 35.13-35.30% | +5.39-5.57 pp | Measuring local train-source utility lift over the anchor does not replace the typed no-op prior; positive-advantage gates over-filter useful residual geometry |
|
| 53 |
-
| K4 mean-by-type residual retrieval + train-family success bonus | No | No | 35.25-35.42% | +5.51-5.68 pp | A continuous train terminal-success prior is below the best by itself and only ties when added to the no-op row; train outcome reliability does not add the gain |
|
| 54 |
| K4 mean-by-type residual retrieval + train-neighbor consensus penalty | No | No | 35.19-35.36% | +5.45-5.62 pp | Penalizing high-dispersion local tangent families is coherent but over-abstains by one success; geometric confidence does not improve the sparse no-op scale-grid row |
|
|
|
|
|
|
|
| 55 |
| K4 repair-tangent residual transport | No | No | 34.14-34.43% | +4.41-4.70 pp | Reversing residuals into failure-to-expert repair tangents is a clean negative diagnostic; the current gain is not recovered by transporting near-miss-to-expert vectors |
|
| 56 |
| K4 mean-by-type residual retrieval + source-score prior 0.025 | No | No | 35.19% | +5.45 pp | A stronger reward-score prior drops below the plateau |
|
| 57 |
| K4 mean-by-type residual retrieval, no-op-only residuals | No | No | 35.19% | +5.45 pp | Removing wrong-gripper residuals loses one success versus the fixed-scale safe-family plateau; the core gain is sparse no-op/tangent repair, with wrong-gripper acting only as a marginal helper |
|
|
@@ -98,34 +101,35 @@ Suggested main-table rows:
|
|
| 98 |
12. K4 train-state residual retrieval, mean-by-type tangent consensus
|
| 99 |
13. K4 mean-by-type residual retrieval + fixed-scale no-op prior plateau, canonical 0.03
|
| 100 |
14. K4 mean-by-type residual retrieval + scale-grid no-op prior 0.03
|
| 101 |
-
15. K4
|
| 102 |
-
16. K4 mean-by-type residual retrieval +
|
| 103 |
-
17. K4 mean-by-type residual retrieval +
|
| 104 |
-
18. K4 mean-by-type residual retrieval +
|
| 105 |
-
19. K4 mean-by-type residual retrieval + train-
|
| 106 |
-
20. K4
|
| 107 |
-
21. K4
|
| 108 |
-
22. K4 mean-by-type residual retrieval +
|
| 109 |
-
23.
|
| 110 |
-
24.
|
| 111 |
-
25. K4
|
| 112 |
-
26. K4
|
| 113 |
-
27. K4
|
| 114 |
-
28.
|
| 115 |
-
29.
|
| 116 |
-
30. Residual
|
| 117 |
-
31.
|
| 118 |
-
32. Lattice,
|
| 119 |
-
33. Lattice, no expert
|
| 120 |
-
34. Lattice,
|
| 121 |
-
35.
|
|
|
|
| 122 |
|
| 123 |
Suggested claim:
|
| 124 |
|
| 125 |
> DoVLA-CIL is not a better behavior-cloning policy; it is a local counterfactual action
|
| 126 |
> selection rule. Deployment-clean K4 consensus residual transport with advantage
|
| 127 |
-
> abstention, a small typed no-op prior,
|
| 128 |
-
>
|
| 129 |
> extending the scale grid upward and adding minimum-energy action regularization
|
| 130 |
> are near-tie/negative, so the effect is local calibration rather than larger or
|
| 131 |
> simply shorter steps. Train-source progress/reward-score priors provide cleaner
|
|
@@ -133,8 +137,10 @@ Suggested claim:
|
|
| 133 |
> suggesting transferable residuals need not beat the expert anchor in their source
|
| 134 |
> state. Continuous train-family success priors likewise tie or drop rather than
|
| 135 |
> explain the top row. A train-neighbor consensus penalty is also negative/near-tie,
|
| 136 |
-
> suggesting the current field already performs most of the useful abstention.
|
| 137 |
-
>
|
|
|
|
|
|
|
| 138 |
> near-miss-to-expert correction vectors is not the missing deployment proposal.
|
| 139 |
> Ungated KNN residual
|
| 140 |
> retrieval, field-gradient ascent, broader non-expert BC targets, field-teacher/tangent distillation, z-score/task-relative retrieval,
|
|
|
|
| 5 |
|
| 6 |
For paired seed deltas, per-task gaps, and selection histograms, regenerate and
|
| 7 |
read `paper_analysis.md` with `python3 scripts/build_paper_analysis.py`. Current
|
| 8 |
+
paired analysis: best clean K4 masked composed type-consensus transport with
|
| 9 |
+
typed no-op prior is `+5.80 pp` over canonical h=16, same-state no-expert
|
| 10 |
+
lattice is `+27.25 pp`, and the remaining clean-to-same-state proposal gap is
|
| 11 |
+
`+21.45 pp`.
|
| 12 |
|
| 13 |
| Method | Uses same-state proposals | Uses expert proposal | Success | Gain vs policy | Interpretation |
|
| 14 |
|---|---:|---:|---:|---:|---|
|
|
|
|
| 42 |
| K2 task-relative residual retrieval, safe residuals + advantage margin 0.20 | No | No | 34.26% | +4.52 pp | Actor-pose-only retrieval is too lossy; raw full-state similarity is better for residual transfer |
|
| 43 |
| K4 train-state residual retrieval, safe residuals + mean-by-type tangent consensus | No | No | 34.96% | +5.22 pp | Near-tie clean diagnostic; consensus alone does not beat raw K2 residuals |
|
| 44 |
| K4 mean-by-type residual retrieval + no-op prior 0.03 | No | No | 35.25% | +5.51 pp | Previous fixed-scale clean plateau; 0.025-0.035 nudges high-value no-op residuals without changing the core proposal family |
|
| 45 |
+
| K4 mean-by-type residual retrieval + scale-grid no-op prior 0.03 | No | No | 35.42% | +5.68 pp | Previous clean best; field-gated tangent length calibration improves the fixed-scale plateau while staying within the same local residual geometry |
|
| 46 |
| K4 mean-by-type residual retrieval + source-progress prior 0.03 | No | No | 35.25% | +5.51 pp | Ties the fixed-scale typed prior without a hand typed no-op prior; train-measured source progress can replace but not improve the typed prior |
|
| 47 |
| K4 mean-by-type residual retrieval + source-progress prior 0.05 | No | No | 35.13% | +5.39 pp | A stronger measured-progress prior over-selects nonzero residuals and drops below the plateau |
|
| 48 |
| K4 mean-by-type residual retrieval + source-score prior 0.015/0.020 | No | No | 35.25% | +5.51 pp | Full train reward score, including terminal success, also replaces the fixed-scale typed prior without improving it |
|
| 49 |
| K4 mean-by-type residual retrieval + scale-grid source-score prior 0.02 | No | No | 35.30% | +5.57 pp | Measured train-source prior benefits from tangent length calibration but remains below the typed no-op scale-grid row |
|
| 50 |
| K4 mean-by-type residual retrieval + upper scale-grid no-op prior 0.03 | No | No | 35.36% | +5.62 pp | Scales 0.40/0.45/0.50 nearly tie but do not beat the 0.35/0.40/0.45 row |
|
| 51 |
| K4 mean-by-type residual retrieval + wide scale-grid no-op prior 0.03 | No | No | 35.13% | +5.39 pp | Including 0.55 over-extends the transported tangent and drops below the best local calibration |
|
| 52 |
+
| K4 mean-by-type residual retrieval + minimum-energy action penalty | No | No | 35.36-35.42% | +5.62-5.68 pp | A tiny action L2 penalty (0.05) ties the previous scale-grid row, while 0.10/0.20 drop slightly; shortest-action regularization does not add the gain |
|
| 53 |
| K4 mean-by-type residual retrieval + source-advantage prior/gate | No | No | 35.13-35.30% | +5.39-5.57 pp | Measuring local train-source utility lift over the anchor does not replace the typed no-op prior; positive-advantage gates over-filter useful residual geometry |
|
| 54 |
+
| K4 mean-by-type residual retrieval + train-family success bonus | No | No | 35.25-35.42% | +5.51-5.68 pp | A continuous train terminal-success prior is below the best by itself and only ties the previous scale-grid row when added to the no-op row; train outcome reliability does not add the gain |
|
| 55 |
| K4 mean-by-type residual retrieval + train-neighbor consensus penalty | No | No | 35.19-35.36% | +5.45-5.62 pp | Penalizing high-dispersion local tangent families is coherent but over-abstains by one success; geometric confidence does not improve the sparse no-op scale-grid row |
|
| 56 |
+
| K4 composed type-consensus residual retrieval, masked | No | No | 35.30% | +5.57 pp | Clean anti-goal composite masking removes the confound; tangent composition alone near-ties but does not beat mean-by-type scale-grid transport |
|
| 57 |
+
| K4 composed type-consensus residual retrieval, masked + no-op prior 0.03 | No | No | 35.54% | +5.80 pp | Current best clean diagnostic; masked local tangent composition adds one success over the previous scale-grid no-op row while staying sparse |
|
| 58 |
| K4 repair-tangent residual transport | No | No | 34.14-34.43% | +4.41-4.70 pp | Reversing residuals into failure-to-expert repair tangents is a clean negative diagnostic; the current gain is not recovered by transporting near-miss-to-expert vectors |
|
| 59 |
| K4 mean-by-type residual retrieval + source-score prior 0.025 | No | No | 35.19% | +5.45 pp | A stronger reward-score prior drops below the plateau |
|
| 60 |
| K4 mean-by-type residual retrieval, no-op-only residuals | No | No | 35.19% | +5.45 pp | Removing wrong-gripper residuals loses one success versus the fixed-scale safe-family plateau; the core gain is sparse no-op/tangent repair, with wrong-gripper acting only as a marginal helper |
|
|
|
|
| 101 |
12. K4 train-state residual retrieval, mean-by-type tangent consensus
|
| 102 |
13. K4 mean-by-type residual retrieval + fixed-scale no-op prior plateau, canonical 0.03
|
| 103 |
14. K4 mean-by-type residual retrieval + scale-grid no-op prior 0.03
|
| 104 |
+
15. K4 masked composed type-consensus residual retrieval + no-op prior 0.03
|
| 105 |
+
16. K4 mean-by-type residual retrieval + upper/wide tangent-length diagnostics
|
| 106 |
+
17. K4 mean-by-type residual retrieval + minimum-energy action penalty diagnostics
|
| 107 |
+
18. K4 mean-by-type residual retrieval + source-progress/source-score/source-advantage prior diagnostics
|
| 108 |
+
19. K4 mean-by-type residual retrieval + train-family success bonus diagnostics
|
| 109 |
+
20. K4 mean-by-type residual retrieval + train-neighbor consensus-confidence diagnostics
|
| 110 |
+
21. K4 repair-tangent residual transport diagnostics
|
| 111 |
+
22. K4 mean-by-type residual retrieval + no-op-only family diagnostic
|
| 112 |
+
23. K4 mean-by-type residual retrieval + abstention margin fine sweep
|
| 113 |
+
24. Source-progress viability gate diagnostics
|
| 114 |
+
25. K2/K4 task-relative retrieval metric diagnostics
|
| 115 |
+
26. K4 kernel-weighted residual consensus + no-op prior diagnostics
|
| 116 |
+
27. K4 field-softmax residual barycenter + margin diagnostics
|
| 117 |
+
28. K4 mean-by-type residual retrieval + wrong-gripper typed-prior diagnostics
|
| 118 |
+
29. K2 broad tangent ray-search
|
| 119 |
+
30. Residual-tangent distillation policy
|
| 120 |
+
31. Residual+Gaussian hybrid, K32 sigma0.35
|
| 121 |
+
32. Lattice, near-miss only
|
| 122 |
+
33. Lattice, no expert
|
| 123 |
+
34. Lattice, no expert + policy baseline candidate
|
| 124 |
+
35. Lattice, full
|
| 125 |
+
36. Oracle ceiling
|
| 126 |
|
| 127 |
Suggested claim:
|
| 128 |
|
| 129 |
> DoVLA-CIL is not a better behavior-cloning policy; it is a local counterfactual action
|
| 130 |
> selection rule. Deployment-clean K4 consensus residual transport with advantage
|
| 131 |
+
> abstention, a small typed no-op prior, field-gated tangent length calibration,
|
| 132 |
+
> and masked local tangent composition gives the strongest clean gain so far;
|
| 133 |
> extending the scale grid upward and adding minimum-energy action regularization
|
| 134 |
> are near-tie/negative, so the effect is local calibration rather than larger or
|
| 135 |
> simply shorter steps. Train-source progress/reward-score priors provide cleaner
|
|
|
|
| 137 |
> suggesting transferable residuals need not beat the expert anchor in their source
|
| 138 |
> state. Continuous train-family success priors likewise tie or drop rather than
|
| 139 |
> explain the top row. A train-neighbor consensus penalty is also negative/near-tie,
|
| 140 |
+
> suggesting the current field already performs most of the useful abstention.
|
| 141 |
+
> The clean composition row improves only after anti-goal composite masking, which
|
| 142 |
+
> supports a controlled local tangent-chart story rather than an uncontrolled
|
| 143 |
+
> proposal pile-up. Repair-tangent transport is negative, showing that simply reversing train failures into
|
| 144 |
> near-miss-to-expert correction vectors is not the missing deployment proposal.
|
| 145 |
> Ungated KNN residual
|
| 146 |
> retrieval, field-gradient ascent, broader non-expert BC targets, field-teacher/tangent distillation, z-score/task-relative retrieval,
|
results/paper_story_memo.md
CHANGED
|
@@ -16,7 +16,7 @@ when queried on proposal geometry that matches those local counterfactuals.
|
|
| 16 |
| Same-state local counterfactual proposals are the mechanism | near-miss-only lattice is 55.94%; removing expert+near_miss drops to 25.57% | Strongly supported |
|
| 17 |
| Conservative same-state result is large | no-expert lattice is 56.99% vs 29.74% policy | Main result |
|
| 18 |
| Full lattice gives upper result | full lattice is 69.33%, oracle is 86.78% | Strong but label expert proposal clearly |
|
| 19 |
-
| Deployment-clean proposal is currently a bottleneck | best clean K4
|
| 20 |
| Gradient-based field optimization does not solve the clean proposal gap | `field_optim` best observed result is 25.39% | Negative diagnostic |
|
| 21 |
| A broader non-expert proposal target does not reduce the proposal gap | direct broad non-expert policy is 27.88%; with field scoring it is 26.49% | Negative diagnostic |
|
| 22 |
| Counterfactual residuals transfer better than absolute retrieved actions | nearest residual retrieval is 32.12% vs absolute retrieval 28.93%; KNN4 residual drops to 29.91% | Supported as a clean bridge |
|
|
@@ -26,27 +26,28 @@ when queried on proposal geometry that matches those local counterfactuals.
|
|
| 26 |
| All-split field-teacher distillation does not fix checkpointing/coverage | allmap direct is 28.00%; field-guided best is 26.49% despite 100% target coverage | Negative diagnostic |
|
| 27 |
| Residual family consistency improves clean transport | policy/no-op/wrong-gripper typed residuals reach 33.74%, above raw 33.33% | Supported as diagnostic |
|
| 28 |
| Counterfactual advantage abstention improves clean transport | requiring field advantage over the zero-residual policy raises typed residual transport to 34.84%, and K2 retrieval reaches 35.01% | Supported as the previous clean best |
|
| 29 |
-
| Clean residual transport behaves like sparse intervention |
|
| 30 |
| Tangent consensus is close but needs sparse typing | K4 mean-by-type residual consensus reaches 34.96%; a small no-op residual prior plateau at 0.025-0.035 raises fixed-scale transport to 35.25% | Fixed-scale clean diagnostic |
|
| 31 |
-
| Field-gated tangent length calibration improves the clean bridge | K4 mean-by-type scale grid 0.35/0.40/0.45 with no-op bonus 0.03 reaches 35.42%; the source-score version reaches 35.30%. Upper 0.40/0.45/0.50 nearly ties at 35.36%, while wide 0.35/0.45/0.55 drops to 35.13% |
|
| 32 |
-
|
|
|
|
|
| 33 |
| Source-advantage priors/gates are too brittle | source-advantage bonuses 0.02/0.05 reach 35.13%; no-op+advantage bonus reaches 35.30%; positive-advantage gates reach 35.13% with or without no-op prior | Negative diagnostic: useful transferable tangents need not beat the expert anchor in their own source state |
|
| 34 |
-
| Continuous train-family success priors do not add the gain | scale-grid family-success bonuses 0.02/0.03/0.05 reach 35.25%; no-op+family-success 0.02 ties the
|
| 35 |
| Train-neighbor consensus confidence does not improve the top row | consensus-only 0.05 reaches 35.19%; no-op+consensus penalties 0.02/0.05/0.10 reach 35.36% | Negative/near-tie diagnostic: residual dispersion is a plausible confidence signal, but the field+margin already abstains better |
|
| 36 |
-
| Repair-tangent transport is not the missing clean proposal | reversing residual direction to build near-miss/failure-to-expert tangents reaches only 34.14-34.43%, below the 35.42% scale-grid no-op row | Negative diagnostic: the failure-to-expert vector hypothesis is cleaner than a new prior, but does not explain the gap |
|
| 37 |
| Kernel-weighted tangent interpolation does not beat equal consensus | K4 kernel-weighted residual consensus reaches 34.96%; with no-op prior and scales 0.35/0.40/0.45 it reaches 35.13%/35.19%/35.19%, below the 35.25% mean-consensus plateau | Negative/near-tie diagnostic |
|
| 38 |
| Field-conditioned tangent barycenters identify good sparse corrections but do not close the proposal gap | K4 field-softmax transport reaches 34.96%; with no-op prior and margins 0.10/0.05/0.00 it reaches 35.19%/35.07%/34.84%. Selected aggregate residuals are high-value (up to 60.00% success), but selecting more of them degrades the global row | Negative/near-tie diagnostic |
|
| 39 |
-
| Tangent ray-search does not beat the typed-prior clean row | K1/K2 tight scale-grid ray search reach 34.84%; K2 broad reaches 34.96%; K4 tight reaches 34.55%, all below the scale-grid mean-consensus row at 35.42% | Near-tie/negative diagnostic |
|
| 40 |
| Typed no-op residual prior improves the clean bridge | CPU smoke `14883591` passed; bonuses 0.025/0.03/0.035 tie at 35.25%, while 0.01/0.02/0.05/0.08 are slightly lower | Fixed-scale clean diagnostic |
|
| 41 |
| Wrong-gripper typed prior does not add a new clean bridge | wrong-gripper-only reaches 35.19%; no-op+wrong-gripper 0.02 ties 35.25%; no-op+wrong-gripper 0.04 drops to 35.13% | Negative/tie diagnostic |
|
| 42 |
| No-op-only residuals nearly preserve the fixed-scale clean bridge | excluding wrong-gripper residuals gives 35.19% with either no-op bonus 0.03 or source-score bonus 0.02, one success below the 35.25% fixed-scale safe-family plateau | Mechanism sharpened: wrong-gripper is marginal, not core |
|
| 43 |
-
| The proposal gap is now quantified | `paper_analysis.md` reports best clean +5.
|
| 44 |
| Policy fallback is not the same-state mechanism | adding a policy baseline candidate to the no-expert same-state lattice drops 56.99% to 40.70% even with margin 0.00 | Negative diagnostic |
|
| 45 |
| Z-score retrieval metric does not help | z-score rows reach 32.23-32.81%, below raw retrieval | Negative diagnostic |
|
| 46 |
| Task-relative actor-pose retrieval metric does not improve tangent transfer | K2 task-relative residual retrieval reaches 34.26% vs raw K2 35.01%; K4 task-relative mean-by-type + no-op reaches 34.43% vs raw K4 35.25% | Negative diagnostic |
|
| 47 |
| Train-source progress viability is too blunt a residual gate | source-progress thresholds 0.25/0.50/0.75 reach 35.19%/34.96%/34.72%, below the unfiltered no-op plateau at 35.25% | Negative/near-tie diagnostic |
|
| 48 |
| Continuous train-source progress prior can replace the fixed-scale typed no-op prior but not improve it | source-progress bonus 0.03 ties the 35.25% fixed-scale row exactly; bonus 0.05 drops to 35.13% | Cleaner tie diagnostic |
|
| 49 |
-
| Full train-source reward-score prior also ties fixed-scale but does not improve the clean best | source-score bonuses 0.015/0.020 tie 35.25%; scale-grid source-score reaches 35.30%, still below no-op scale-grid at 35.42% | Cleaner near-tie diagnostic |
|
| 50 |
| Advantage margin 0.20 is a local optimum for K4 tangent consensus | no-op prior margins 0.15/0.20/0.25 reach 35.07%/35.25%/34.84%; source-score prior margins reach 34.96%/35.25%/34.84% | Abstention plateau sharpened |
|
| 51 |
| Train-split residual family reliability does not recover the typed mask | after fixing threshold pass-through, scale-0.35 thresholds 0.10/0.25 reach 33.33%/33.28%, below typed safe residuals | Negative diagnostic |
|
| 52 |
| Residual-tangent distillation does not solve clean proposal generation | aligned allmap tangent student reaches 28.87% despite low pseudo-target BC loss | Negative diagnostic |
|
|
@@ -74,33 +75,34 @@ clean proposal result, the intended main rows are:
|
|
| 74 |
14. K4 mean-by-type tangent consensus: 34.96%
|
| 75 |
15. K4 mean-by-type tangent consensus + typed no-op prior 0.025-0.035: 35.25%
|
| 76 |
16. K4 mean-by-type tangent consensus + scale-grid typed no-op prior: 35.42%
|
| 77 |
-
17. K4
|
| 78 |
-
18. K4 mean-by-type tangent consensus +
|
| 79 |
-
19. K4 mean-by-type tangent consensus +
|
| 80 |
-
20. K4 mean-by-type tangent consensus + train-source
|
| 81 |
-
21. K4 mean-by-type tangent consensus + train-source
|
| 82 |
-
22. K4 mean-by-type tangent consensus + train-
|
| 83 |
-
23. K4 mean-by-type tangent consensus + train-
|
| 84 |
-
24. K4
|
| 85 |
-
25. K4
|
| 86 |
-
26. K4 mean-by-type
|
| 87 |
-
27.
|
| 88 |
-
28.
|
| 89 |
-
29. K4
|
| 90 |
-
30.
|
| 91 |
-
31.
|
| 92 |
-
32.
|
| 93 |
-
33.
|
| 94 |
-
34.
|
| 95 |
-
35.
|
| 96 |
-
36.
|
| 97 |
-
37.
|
| 98 |
-
38.
|
| 99 |
-
39.
|
| 100 |
-
40. Lattice,
|
| 101 |
-
41. Lattice, no expert
|
| 102 |
-
42. Lattice,
|
| 103 |
-
43.
|
|
|
|
| 104 |
|
| 105 |
## Novelty Framing
|
| 106 |
|
|
@@ -128,13 +130,13 @@ test-time search. The cleaner novelty is:
|
|
| 128 |
|
| 129 |
## Job Status
|
| 130 |
|
| 131 |
-
Last checked: `2026-06-29
|
| 132 |
-
completed and produced a new clean best, 35.
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
|
| 139 |
- `14858328`-`14858333`: completed train-split `field_selected_noexpert_bc5`;
|
| 140 |
direct rollout is 26.84%, field-guided best is 27.65%.
|
|
@@ -250,14 +252,14 @@ to use the scale-grid no-op row as `best_clean_key`.
|
|
| 250 |
- `14903128`/`14903130`/`14903132`/`14903134`: completed continuous
|
| 251 |
train-family success-prior GPU arrays. Family-success bonuses `0.02`, `0.03`,
|
| 252 |
and `0.05` reach 35.25%; adding family-success `0.02` to the no-op `0.03`
|
| 253 |
-
|
| 254 |
`14903131`/`14903133`/`14903135` and rebuild job `14903136` completed.
|
| 255 |
- `14903296`: completed CPU smoke for the train-neighbor consensus-confidence
|
| 256 |
penalty path, validating metadata and Slurm/CLI wiring.
|
| 257 |
- `14903384`/`14903386`/`14903388`/`14903390`: completed consensus-confidence
|
| 258 |
GPU arrays. Consensus-only `0.05` reaches 35.19%; no-op `0.03` plus
|
| 259 |
consensus penalties `0.02`, `0.05`, and `0.10` all reach 35.36%, one success
|
| 260 |
-
below the 35.42% best. Summary jobs `14903385`/`14903387`/`14903389`/
|
| 261 |
`14903391` and rebuild job `14903392` completed.
|
| 262 |
- `14904575`: completed CPU smoke for repair-tangent residual direction
|
| 263 |
(`anchor_minus_candidate`). The smoke wrote valid metadata and selected the
|
|
@@ -267,6 +269,21 @@ to use the scale-grid no-op row as `best_clean_key`.
|
|
| 267 |
repair row reaches 34.43%. Summary jobs `14904738`/`14904741`/`14904743`/
|
| 268 |
`14904745` completed, local paper builders updated the artifacts, and the
|
| 269 |
queued rebuild job `14904803` was canceled after local rebuilds finished.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 270 |
- `14894281`: completed the Apptainer unit smoke for the train-source
|
| 271 |
progress-viability gate, including the variable residual-count padding check
|
| 272 |
(`source_progress_lengths == [3, 3]`).
|
|
@@ -284,14 +301,14 @@ to use the scale-grid no-op row as `best_clean_key`.
|
|
| 284 |
train-source progress bonus path. The unit smoke validated bonus padding
|
| 285 |
(`source_progress_bonuses == [[0, 0.08, 0], [0, 0.08, 0.08]]`).
|
| 286 |
- `14894674`/`14894675`: completed source-progress bonus arrays with no fixed
|
| 287 |
-
no-op prior. Bonus `0.03` ties the
|
| 288 |
reaches 35.13%. Summary jobs `14894676`/`14894677` completed; rebuild job
|
| 289 |
`14894678` was queued after them.
|
| 290 |
- `14897121`/`14897122`: completed unit and CPU rollout smokes for the
|
| 291 |
train-source reward-score bonus path. The unit smoke validates that terminal
|
| 292 |
success contributes to the candidate prior.
|
| 293 |
- `14897123`/`14897124`/`14897125`: completed source-score bonus arrays.
|
| 294 |
-
Bonuses `0.015` and `0.020` tie the
|
| 295 |
reaches 35.19%. Summary jobs `14897126`/`14897127`/`14897128` and rebuild job
|
| 296 |
`14897129` completed.
|
| 297 |
- `14897548`/`14897549`: completed no-op-only CPU rollout smokes after excluding
|
|
@@ -308,15 +325,15 @@ to use the scale-grid no-op row as `best_clean_key`.
|
|
| 308 |
`14897845`-`14897848` and rebuild job `14897849` completed.
|
| 309 |
- `14897988`/`14897989`: completed K4 mean-by-type scale-grid sweeps using
|
| 310 |
scales `0.35/0.40/0.45`, margin `0.20`, and safe residual families. The typed
|
| 311 |
-
no-op prior row reaches a new clean best, 35.42%; the source-score prior row
|
| 312 |
reaches 35.30%. Summary jobs `14897990`/`14897991` completed; rebuild job
|
| 313 |
`14897992` was submitted, and local rebuilds updated the paper artifacts.
|
| 314 |
- `14898107`/`14898108`/`14898109`: completed upper and wide K4 mean-by-type
|
| 315 |
scale-grid follow-ups. The no-op upper grid `0.40/0.45/0.50` reaches 35.36%,
|
| 316 |
the source-score upper grid reaches 35.30%, and the no-op wide grid
|
| 317 |
`0.35/0.45/0.55` reaches 35.13%. Summary jobs `14898110`/`14898111`/
|
| 318 |
-
`14898112` and rebuild job `14898113` completed; the best clean
|
| 319 |
-
the `0.35/0.40/0.45` no-op grid at 35.42%.
|
| 320 |
- `14898293`: completed the CPU Apptainer smoke for the residual action-L2
|
| 321 |
penalty path with the best scale-grid/no-op configuration.
|
| 322 |
- `14898327`/`14898329`/`14898331`: completed minimum-energy tangent GPU sweeps
|
|
@@ -350,10 +367,13 @@ to use the scale-grid no-op row as `best_clean_key`.
|
|
| 350 |
|
| 351 |
- Promote same-state no-expert lattice (56.99%) as the conservative mechanism
|
| 352 |
result.
|
| 353 |
-
- Use K4
|
| 354 |
-
typed no-op prior, and field-gated tangent length
|
| 355 |
-
`0.35/0.40/0.45` as the current best clean deployment
|
| 356 |
-
as a SOTA claim. The
|
|
|
|
|
|
|
|
|
|
| 357 |
train-source progress/reward-score priors tie that fixed-scale row, and
|
| 358 |
scale-grid source-score reaches 35.30% but not the new best. Source-advantage
|
| 359 |
priors/gates reach at most 35.30%, so local utility lift over the source
|
|
@@ -374,6 +394,6 @@ to use the scale-grid no-op row as `best_clean_key`.
|
|
| 374 |
repaired train-family reliability priors, Gaussian hybrids,
|
| 375 |
field optimization, field-teacher/tangent distillation, repair-tangent transport, policy-relative anchoring, tangent consensus,
|
| 376 |
kernel-weighted tangent interpolation, field-softmax tangent barycenters,
|
| 377 |
-
wrong-gripper typed priors, and
|
| 378 |
-
|
| 379 |
-
proposal geometry.
|
|
|
|
| 16 |
| Same-state local counterfactual proposals are the mechanism | near-miss-only lattice is 55.94%; removing expert+near_miss drops to 25.57% | Strongly supported |
|
| 17 |
| Conservative same-state result is large | no-expert lattice is 56.99% vs 29.74% policy | Main result |
|
| 18 |
| Full lattice gives upper result | full lattice is 69.33%, oracle is 86.78% | Strong but label expert proposal clearly |
|
| 19 |
+
| Deployment-clean proposal is currently a bottleneck | best clean K4 masked composed type-consensus transport with a small typed no-op prior is 35.54%, far below 56.99% | Supported |
|
| 20 |
| Gradient-based field optimization does not solve the clean proposal gap | `field_optim` best observed result is 25.39% | Negative diagnostic |
|
| 21 |
| A broader non-expert proposal target does not reduce the proposal gap | direct broad non-expert policy is 27.88%; with field scoring it is 26.49% | Negative diagnostic |
|
| 22 |
| Counterfactual residuals transfer better than absolute retrieved actions | nearest residual retrieval is 32.12% vs absolute retrieval 28.93%; KNN4 residual drops to 29.91% | Supported as a clean bridge |
|
|
|
|
| 26 |
| All-split field-teacher distillation does not fix checkpointing/coverage | allmap direct is 28.00%; field-guided best is 26.49% despite 100% target coverage | Negative diagnostic |
|
| 27 |
| Residual family consistency improves clean transport | policy/no-op/wrong-gripper typed residuals reach 33.74%, above raw 33.33% | Supported as diagnostic |
|
| 28 |
| Counterfactual advantage abstention improves clean transport | requiring field advantage over the zero-residual policy raises typed residual transport to 34.84%, and K2 retrieval reaches 35.01% | Supported as the previous clean best |
|
| 29 |
+
| Clean residual transport behaves like sparse intervention | the best clean row abstains to zero-residual policy on 93.2% of states, while selected nonzero residuals succeed at 49.6% vs 34.5% for abstention | Stronger clean-mechanism framing |
|
| 30 |
| Tangent consensus is close but needs sparse typing | K4 mean-by-type residual consensus reaches 34.96%; a small no-op residual prior plateau at 0.025-0.035 raises fixed-scale transport to 35.25% | Fixed-scale clean diagnostic |
|
| 31 |
+
| Field-gated tangent length calibration improves the clean bridge | K4 mean-by-type scale grid 0.35/0.40/0.45 with no-op bonus 0.03 reaches 35.42%; the source-score version reaches 35.30%. Upper 0.40/0.45/0.50 nearly ties at 35.36%, while wide 0.35/0.45/0.55 drops to 35.13% | Previous clean best; local scale calibration, not a larger-step effect |
|
| 32 |
+
| Masked tangent composition gives a small clean lift | K4 composed type-consensus transport is 35.30% after anti-goal composite masking, and reaches 35.54% with the typed no-op prior; raw selected types contain no random-negative or wrong-direction composites | Current best clean result; controlled local tangent-chart composition, not unmasked proposal accumulation |
|
| 33 |
+
| Minimum-energy residual regularization does not add the gain | action L2 penalty 0.05 ties the previous 35.42% scale-grid row, while 0.10/0.20 reach 35.36% | Negative/tie diagnostic: the clean bridge is not explained by shortest-action bias |
|
| 34 |
| Source-advantage priors/gates are too brittle | source-advantage bonuses 0.02/0.05 reach 35.13%; no-op+advantage bonus reaches 35.30%; positive-advantage gates reach 35.13% with or without no-op prior | Negative diagnostic: useful transferable tangents need not beat the expert anchor in their own source state |
|
| 35 |
+
| Continuous train-family success priors do not add the gain | scale-grid family-success bonuses 0.02/0.03/0.05 reach 35.25%; no-op+family-success 0.02 ties the previous scale-grid row at 35.42% | Negative/tie diagnostic: train terminal success is not the right confidence signal for transferred tangents |
|
| 36 |
| Train-neighbor consensus confidence does not improve the top row | consensus-only 0.05 reaches 35.19%; no-op+consensus penalties 0.02/0.05/0.10 reach 35.36% | Negative/near-tie diagnostic: residual dispersion is a plausible confidence signal, but the field+margin already abstains better |
|
| 37 |
+
| Repair-tangent transport is not the missing clean proposal | reversing residual direction to build near-miss/failure-to-expert tangents reaches only 34.14-34.43%, below the previous 35.42% scale-grid no-op row | Negative diagnostic: the failure-to-expert vector hypothesis is cleaner than a new prior, but does not explain the gap |
|
| 38 |
| Kernel-weighted tangent interpolation does not beat equal consensus | K4 kernel-weighted residual consensus reaches 34.96%; with no-op prior and scales 0.35/0.40/0.45 it reaches 35.13%/35.19%/35.19%, below the 35.25% mean-consensus plateau | Negative/near-tie diagnostic |
|
| 39 |
| Field-conditioned tangent barycenters identify good sparse corrections but do not close the proposal gap | K4 field-softmax transport reaches 34.96%; with no-op prior and margins 0.10/0.05/0.00 it reaches 35.19%/35.07%/34.84%. Selected aggregate residuals are high-value (up to 60.00% success), but selecting more of them degrades the global row | Negative/near-tie diagnostic |
|
| 40 |
+
| Tangent ray-search does not beat the typed-prior clean row | K1/K2 tight scale-grid ray search reach 34.84%; K2 broad reaches 34.96%; K4 tight reaches 34.55%, all below the previous scale-grid mean-consensus row at 35.42% | Near-tie/negative diagnostic |
|
| 41 |
| Typed no-op residual prior improves the clean bridge | CPU smoke `14883591` passed; bonuses 0.025/0.03/0.035 tie at 35.25%, while 0.01/0.02/0.05/0.08 are slightly lower | Fixed-scale clean diagnostic |
|
| 42 |
| Wrong-gripper typed prior does not add a new clean bridge | wrong-gripper-only reaches 35.19%; no-op+wrong-gripper 0.02 ties 35.25%; no-op+wrong-gripper 0.04 drops to 35.13% | Negative/tie diagnostic |
|
| 43 |
| No-op-only residuals nearly preserve the fixed-scale clean bridge | excluding wrong-gripper residuals gives 35.19% with either no-op bonus 0.03 or source-score bonus 0.02, one success below the 35.25% fixed-scale safe-family plateau | Mechanism sharpened: wrong-gripper is marginal, not core |
|
| 44 |
+
| The proposal gap is now quantified | `paper_analysis.md` reports best clean +5.80 pp over canonical h16, same-state no-expert +27.25 pp, leaving a +21.45 pp clean-to-same-state gap | Core paper tension |
|
| 45 |
| Policy fallback is not the same-state mechanism | adding a policy baseline candidate to the no-expert same-state lattice drops 56.99% to 40.70% even with margin 0.00 | Negative diagnostic |
|
| 46 |
| Z-score retrieval metric does not help | z-score rows reach 32.23-32.81%, below raw retrieval | Negative diagnostic |
|
| 47 |
| Task-relative actor-pose retrieval metric does not improve tangent transfer | K2 task-relative residual retrieval reaches 34.26% vs raw K2 35.01%; K4 task-relative mean-by-type + no-op reaches 34.43% vs raw K4 35.25% | Negative diagnostic |
|
| 48 |
| Train-source progress viability is too blunt a residual gate | source-progress thresholds 0.25/0.50/0.75 reach 35.19%/34.96%/34.72%, below the unfiltered no-op plateau at 35.25% | Negative/near-tie diagnostic |
|
| 49 |
| Continuous train-source progress prior can replace the fixed-scale typed no-op prior but not improve it | source-progress bonus 0.03 ties the 35.25% fixed-scale row exactly; bonus 0.05 drops to 35.13% | Cleaner tie diagnostic |
|
| 50 |
+
| Full train-source reward-score prior also ties fixed-scale but does not improve the clean best | source-score bonuses 0.015/0.020 tie 35.25%; scale-grid source-score reaches 35.30%, still below the previous no-op scale-grid row at 35.42% | Cleaner near-tie diagnostic |
|
| 51 |
| Advantage margin 0.20 is a local optimum for K4 tangent consensus | no-op prior margins 0.15/0.20/0.25 reach 35.07%/35.25%/34.84%; source-score prior margins reach 34.96%/35.25%/34.84% | Abstention plateau sharpened |
|
| 52 |
| Train-split residual family reliability does not recover the typed mask | after fixing threshold pass-through, scale-0.35 thresholds 0.10/0.25 reach 33.33%/33.28%, below typed safe residuals | Negative diagnostic |
|
| 53 |
| Residual-tangent distillation does not solve clean proposal generation | aligned allmap tangent student reaches 28.87% despite low pseudo-target BC loss | Negative diagnostic |
|
|
|
|
| 75 |
14. K4 mean-by-type tangent consensus: 34.96%
|
| 76 |
15. K4 mean-by-type tangent consensus + typed no-op prior 0.025-0.035: 35.25%
|
| 77 |
16. K4 mean-by-type tangent consensus + scale-grid typed no-op prior: 35.42%
|
| 78 |
+
17. K4 masked composed type-consensus tangent transport: 35.30%; with typed no-op prior: 35.54%
|
| 79 |
+
18. K4 mean-by-type tangent consensus + upper/wide scale diagnostics: 35.36% for 0.40/0.45/0.50; 35.13% for 0.35/0.45/0.55
|
| 80 |
+
19. K4 mean-by-type tangent consensus + action L2 penalty: 35.42% at 0.05; 35.36% at 0.10/0.20
|
| 81 |
+
20. K4 mean-by-type tangent consensus + train-source progress prior: 35.25% at bonus 0.03; 35.13% at bonus 0.05
|
| 82 |
+
21. K4 mean-by-type tangent consensus + train-source reward-score prior: 35.25% at bonuses 0.015/0.020; 35.30% with scale grid; 35.19% at 0.025
|
| 83 |
+
22. K4 mean-by-type tangent consensus + train-source advantage prior/gate: 35.13% at bonuses 0.02/0.05; 35.30% with no-op+advantage; 35.13% with positive-advantage gates
|
| 84 |
+
23. K4 mean-by-type tangent consensus + train-family success bonus: 35.25% alone; 35.42% with no-op bonus 0.03
|
| 85 |
+
24. K4 mean-by-type tangent consensus + train-neighbor consensus penalty: 35.19% alone; 35.36% with no-op bonus 0.03
|
| 86 |
+
25. K4 repair-tangent transport: 34.14-34.43%
|
| 87 |
+
26. K4 mean-by-type tangent consensus, no-op-only residuals: 35.19% with either no-op bonus 0.03 or source-score bonus 0.02
|
| 88 |
+
27. K4 mean-by-type abstention margin sweep: 35.07% / 35.25% / 34.84% for typed no-op margins 0.15 / 0.20 / 0.25; 34.96% / 35.25% / 34.84% for source-score margins
|
| 89 |
+
28. Source-progress viability gates: 35.19% / 34.96% / 34.72% for thresholds 0.25 / 0.50 / 0.75
|
| 90 |
+
29. K4 kernel-weighted tangent consensus / + no-op prior: 34.96% / 35.19%
|
| 91 |
+
30. K4 field-softmax tangent transport / best margin sweep: 34.96% / 35.19%
|
| 92 |
+
31. Wrong-gripper prior / no-op+wrong-gripper prior: 35.19% / 35.25%
|
| 93 |
+
32. K2 broad tangent ray-search: 34.96%
|
| 94 |
+
33. K1/K2 tight tangent ray-search: 34.84% / 34.84%
|
| 95 |
+
34. K4 tight tangent ray-search: 34.55%
|
| 96 |
+
35. Residual-tangent distillation policy: 28.87%
|
| 97 |
+
36. Z-score residual retrieval: 32.23-32.81%
|
| 98 |
+
37. Task-relative residual retrieval metric: 34.26-34.43%
|
| 99 |
+
38. Train-family reliability prior: 33.28-33.33%
|
| 100 |
+
39. Residual+Gaussian hybrid K32/K64: 31.30% / 30.90%
|
| 101 |
+
40. Lattice, near-miss only: 55.94%
|
| 102 |
+
41. Lattice, no expert: 56.99%
|
| 103 |
+
42. Lattice, no expert + policy baseline candidate: 40.70%
|
| 104 |
+
43. Lattice, full: 69.33%
|
| 105 |
+
44. Oracle ceiling: 86.78%
|
| 106 |
|
| 107 |
## Novelty Framing
|
| 108 |
|
|
|
|
| 130 |
|
| 131 |
## Job Status
|
| 132 |
|
| 133 |
+
Last checked: `2026-06-29 16:12 UTC`. The K4 masked composed type-consensus
|
| 134 |
+
sweep completed and produced a new clean best, 35.54%, while the pure masked
|
| 135 |
+
composition row reached 35.30%. Raw selected candidate types show no
|
| 136 |
+
random-negative or wrong-direction composite leak. The paper table and paired
|
| 137 |
+
analysis now use the masked composed no-op row as `best_clean_key`. A
|
| 138 |
+
component-wise composite prior follow-up has passed CPU smoke and is queued for
|
| 139 |
+
3-seed GPU rollout.
|
| 140 |
|
| 141 |
- `14858328`-`14858333`: completed train-split `field_selected_noexpert_bc5`;
|
| 142 |
direct rollout is 26.84%, field-guided best is 27.65%.
|
|
|
|
| 252 |
- `14903128`/`14903130`/`14903132`/`14903134`: completed continuous
|
| 253 |
train-family success-prior GPU arrays. Family-success bonuses `0.02`, `0.03`,
|
| 254 |
and `0.05` reach 35.25%; adding family-success `0.02` to the no-op `0.03`
|
| 255 |
+
row ties the previous 35.42% scale-grid result without adding a new gain. Summary jobs `14903129`/
|
| 256 |
`14903131`/`14903133`/`14903135` and rebuild job `14903136` completed.
|
| 257 |
- `14903296`: completed CPU smoke for the train-neighbor consensus-confidence
|
| 258 |
penalty path, validating metadata and Slurm/CLI wiring.
|
| 259 |
- `14903384`/`14903386`/`14903388`/`14903390`: completed consensus-confidence
|
| 260 |
GPU arrays. Consensus-only `0.05` reaches 35.19%; no-op `0.03` plus
|
| 261 |
consensus penalties `0.02`, `0.05`, and `0.10` all reach 35.36%, one success
|
| 262 |
+
below the previous 35.42% scale-grid best. Summary jobs `14903385`/`14903387`/`14903389`/
|
| 263 |
`14903391` and rebuild job `14903392` completed.
|
| 264 |
- `14904575`: completed CPU smoke for repair-tangent residual direction
|
| 265 |
(`anchor_minus_candidate`). The smoke wrote valid metadata and selected the
|
|
|
|
| 269 |
repair row reaches 34.43%. Summary jobs `14904738`/`14904741`/`14904743`/
|
| 270 |
`14904745` completed, local paper builders updated the artifacts, and the
|
| 271 |
queued rebuild job `14904803` was canceled after local rebuilds finished.
|
| 272 |
+
- `14911977`: completed CPU smoke for masked composed type-consensus transport.
|
| 273 |
+
The smoke selected only `policy_residual` on 8 groups and confirmed composite
|
| 274 |
+
candidate masking excludes random-negative and wrong-direction parts.
|
| 275 |
+
- `14911979`/`14911980`: completed K4 masked composed type-consensus GPU arrays.
|
| 276 |
+
Pure masked composition reaches 35.30%; adding the typed no-op prior reaches
|
| 277 |
+
the new clean best, 35.54%. Raw selected candidate types contain no
|
| 278 |
+
random-negative or wrong-direction composites. Summary jobs `14911982`/
|
| 279 |
+
`14911983` and rebuild job `14911984` completed.
|
| 280 |
+
- `14912552`: completed CPU smoke for component-wise candidate-type bonuses on
|
| 281 |
+
masked composed type-consensus transport. Metadata records
|
| 282 |
+
`candidate_type_bonus_components=True`, selected candidate types have no
|
| 283 |
+
random-negative or wrong-direction leak, and the smoke selected
|
| 284 |
+
`policy_residual` on 8/8 groups.
|
| 285 |
+
- `14912561`/`14912562`/`14912563`: queued component-wise composite-prior GPU
|
| 286 |
+
array, summary, and paper-artifact rebuild behind the passed smoke.
|
| 287 |
- `14894281`: completed the Apptainer unit smoke for the train-source
|
| 288 |
progress-viability gate, including the variable residual-count padding check
|
| 289 |
(`source_progress_lengths == [3, 3]`).
|
|
|
|
| 301 |
train-source progress bonus path. The unit smoke validated bonus padding
|
| 302 |
(`source_progress_bonuses == [[0, 0.08, 0], [0, 0.08, 0.08]]`).
|
| 303 |
- `14894674`/`14894675`: completed source-progress bonus arrays with no fixed
|
| 304 |
+
no-op prior. Bonus `0.03` ties the fixed-scale plateau at 35.25%; bonus `0.05`
|
| 305 |
reaches 35.13%. Summary jobs `14894676`/`14894677` completed; rebuild job
|
| 306 |
`14894678` was queued after them.
|
| 307 |
- `14897121`/`14897122`: completed unit and CPU rollout smokes for the
|
| 308 |
train-source reward-score bonus path. The unit smoke validates that terminal
|
| 309 |
success contributes to the candidate prior.
|
| 310 |
- `14897123`/`14897124`/`14897125`: completed source-score bonus arrays.
|
| 311 |
+
Bonuses `0.015` and `0.020` tie the fixed-scale plateau at 35.25%; bonus `0.025`
|
| 312 |
reaches 35.19%. Summary jobs `14897126`/`14897127`/`14897128` and rebuild job
|
| 313 |
`14897129` completed.
|
| 314 |
- `14897548`/`14897549`: completed no-op-only CPU rollout smokes after excluding
|
|
|
|
| 325 |
`14897845`-`14897848` and rebuild job `14897849` completed.
|
| 326 |
- `14897988`/`14897989`: completed K4 mean-by-type scale-grid sweeps using
|
| 327 |
scales `0.35/0.40/0.45`, margin `0.20`, and safe residual families. The typed
|
| 328 |
+
no-op prior row reaches a then-new clean best, 35.42%; the source-score prior row
|
| 329 |
reaches 35.30%. Summary jobs `14897990`/`14897991` completed; rebuild job
|
| 330 |
`14897992` was submitted, and local rebuilds updated the paper artifacts.
|
| 331 |
- `14898107`/`14898108`/`14898109`: completed upper and wide K4 mean-by-type
|
| 332 |
scale-grid follow-ups. The no-op upper grid `0.40/0.45/0.50` reaches 35.36%,
|
| 333 |
the source-score upper grid reaches 35.30%, and the no-op wide grid
|
| 334 |
`0.35/0.45/0.55` reaches 35.13%. Summary jobs `14898110`/`14898111`/
|
| 335 |
+
`14898112` and rebuild job `14898113` completed; at that stage the best clean
|
| 336 |
+
row remained the `0.35/0.40/0.45` no-op grid at 35.42%.
|
| 337 |
- `14898293`: completed the CPU Apptainer smoke for the residual action-L2
|
| 338 |
penalty path with the best scale-grid/no-op configuration.
|
| 339 |
- `14898327`/`14898329`/`14898331`: completed minimum-energy tangent GPU sweeps
|
|
|
|
| 367 |
|
| 368 |
- Promote same-state no-expert lattice (56.99%) as the conservative mechanism
|
| 369 |
result.
|
| 370 |
+
- Use K4 masked composed type-consensus residual transport with advantage
|
| 371 |
+
abstention, a small typed no-op prior, and field-gated tangent length
|
| 372 |
+
calibration over `0.35/0.40/0.45` as the current best clean deployment
|
| 373 |
+
diagnostic, 35.54%, not as a SOTA claim. The previous mean-by-type scale-grid
|
| 374 |
+
no-op row remains 35.42%; pure masked composition is 35.30%, so composition
|
| 375 |
+
helps only when the sparse typed prior is preserved and anti-goal composite
|
| 376 |
+
parts are masked. The fixed-scale no-op plateau remains 35.25%; continuous
|
| 377 |
train-source progress/reward-score priors tie that fixed-scale row, and
|
| 378 |
scale-grid source-score reaches 35.30% but not the new best. Source-advantage
|
| 379 |
priors/gates reach at most 35.30%, so local utility lift over the source
|
|
|
|
| 394 |
repaired train-family reliability priors, Gaussian hybrids,
|
| 395 |
field optimization, field-teacher/tangent distillation, repair-tangent transport, policy-relative anchoring, tangent consensus,
|
| 396 |
kernel-weighted tangent interpolation, field-softmax tangent barycenters,
|
| 397 |
+
unmasked or prior-free tangent composition, wrong-gripper typed priors, and
|
| 398 |
+
same-state policy-baseline fallback as negative or near-tie diagnostics that
|
| 399 |
+
sharpen the story around local counterfactual proposal geometry.
|
results/paper_table_status.json
CHANGED
|
@@ -1296,6 +1296,25 @@
|
|
| 1296 |
"best_config": null,
|
| 1297 |
"gain_vs_h16_policy": 0.05797101449275366
|
| 1298 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1299 |
{
|
| 1300 |
"key": "retrieval_repair_nearmiss_k4_grid025035050_margin020",
|
| 1301 |
"label": "K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20",
|
|
|
|
| 1296 |
"best_config": null,
|
| 1297 |
"gain_vs_h16_policy": 0.05797101449275366
|
| 1298 |
},
|
| 1299 |
+
{
|
| 1300 |
+
"key": "retrieval_residual_k4_composemasked_compbonus_grid035040045_noopbonus003",
|
| 1301 |
+
"label": "K4 composed type-consensus residual retrieval, masked, component no-op bonus 0.03",
|
| 1302 |
+
"path": "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_compbonus_grid035040045_safe_margin0p20_noopbonus0p03_summary.json",
|
| 1303 |
+
"clean_deployment": "yes",
|
| 1304 |
+
"same_state_proposals": "no",
|
| 1305 |
+
"expert_proposal": "no",
|
| 1306 |
+
"story_role": "component-wise sparse prior on the masked local tangent composition chart",
|
| 1307 |
+
"fallback_success": null,
|
| 1308 |
+
"pending_job": "14912561/14912562",
|
| 1309 |
+
"path_exists": false,
|
| 1310 |
+
"status": "pending",
|
| 1311 |
+
"success": null,
|
| 1312 |
+
"std_success": null,
|
| 1313 |
+
"completed_seeds": null,
|
| 1314 |
+
"num_completed": null,
|
| 1315 |
+
"best_config": null,
|
| 1316 |
+
"gain_vs_h16_policy": null
|
| 1317 |
+
},
|
| 1318 |
{
|
| 1319 |
"key": "retrieval_repair_nearmiss_k4_grid025035050_margin020",
|
| 1320 |
"label": "K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20",
|
results/paper_table_status.md
CHANGED
|
@@ -71,6 +71,7 @@ Baseline h=16 policy: 29.74%
|
|
| 71 |
| retrieval_residual_k4_compose_grid035040045_noopbonus003 | K4 composed type-consensus residual retrieval, scales 0.35/0.40/0.45, margin 0.20, no-op bonus 0.03 | complete | 34.14% | +4.41 pp | yes | no | no | local tangent composition on the current best typed prior |
|
| 72 |
| retrieval_residual_k4_composemasked_grid035040045 | K4 composed type-consensus residual retrieval, masked, scales 0.35/0.40/0.45, margin 0.20 | complete | 35.30% | +5.57 pp | yes | no | no | local tangent composition with anti-goal composite masks |
|
| 73 |
| retrieval_residual_k4_composemasked_grid035040045_noopbonus003 | K4 composed type-consensus residual retrieval, masked, scales 0.35/0.40/0.45, margin 0.20, no-op bonus 0.03 | complete | 35.54% | +5.80 pp | yes | no | no | local tangent composition with anti-goal composite masks on the current best typed prior |
|
|
|
|
| 74 |
| retrieval_repair_nearmiss_k4_grid025035050_margin020 | K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20 | complete | 34.32% | +4.58 pp | yes | no | no | deployment-clean corrective tangent transport from train near-misses back toward expert actions |
|
| 75 |
| retrieval_repair_nearmiss_k4_grid035050075_margin020 | K4 near-miss-to-expert repair tangent, scales 0.35/0.50/0.75, margin 0.20 | complete | 34.38% | +4.64 pp | yes | no | no | repair-tangent scale diagnostic for near-miss counterfactual geometry |
|
| 76 |
| retrieval_repair_nearmiss_k4_grid025035050_margin010 | K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.10 | complete | 34.14% | +4.41 pp | yes | no | no | repair-tangent abstention diagnostic for near-miss counterfactual geometry |
|
|
|
|
| 71 |
| retrieval_residual_k4_compose_grid035040045_noopbonus003 | K4 composed type-consensus residual retrieval, scales 0.35/0.40/0.45, margin 0.20, no-op bonus 0.03 | complete | 34.14% | +4.41 pp | yes | no | no | local tangent composition on the current best typed prior |
|
| 72 |
| retrieval_residual_k4_composemasked_grid035040045 | K4 composed type-consensus residual retrieval, masked, scales 0.35/0.40/0.45, margin 0.20 | complete | 35.30% | +5.57 pp | yes | no | no | local tangent composition with anti-goal composite masks |
|
| 73 |
| retrieval_residual_k4_composemasked_grid035040045_noopbonus003 | K4 composed type-consensus residual retrieval, masked, scales 0.35/0.40/0.45, margin 0.20, no-op bonus 0.03 | complete | 35.54% | +5.80 pp | yes | no | no | local tangent composition with anti-goal composite masks on the current best typed prior |
|
| 74 |
+
| retrieval_residual_k4_composemasked_compbonus_grid035040045_noopbonus003 | K4 composed type-consensus residual retrieval, masked, component no-op bonus 0.03 | pending 14912561/14912562 | pending | pending | yes | no | no | component-wise sparse prior on the masked local tangent composition chart |
|
| 75 |
| retrieval_repair_nearmiss_k4_grid025035050_margin020 | K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20 | complete | 34.32% | +4.58 pp | yes | no | no | deployment-clean corrective tangent transport from train near-misses back toward expert actions |
|
| 76 |
| retrieval_repair_nearmiss_k4_grid035050075_margin020 | K4 near-miss-to-expert repair tangent, scales 0.35/0.50/0.75, margin 0.20 | complete | 34.38% | +4.64 pp | yes | no | no | repair-tangent scale diagnostic for near-miss counterfactual geometry |
|
| 77 |
| retrieval_repair_nearmiss_k4_grid025035050_margin010 | K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.10 | complete | 34.14% | +4.41 pp | yes | no | no | repair-tangent abstention diagnostic for near-miss counterfactual geometry |
|
scripts/build_paper_analysis.py
CHANGED
|
@@ -353,6 +353,14 @@ METHODS = [
|
|
| 353 |
"k4_composemasked_grid035040045_safe_margin0p20_noopbonus0p03_summary.json"
|
| 354 |
),
|
| 355 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 356 |
MethodSpec(
|
| 357 |
key="repair_nearmiss_k4_grid025035050_margin020",
|
| 358 |
label="K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20",
|
|
|
|
| 353 |
"k4_composemasked_grid035040045_safe_margin0p20_noopbonus0p03_summary.json"
|
| 354 |
),
|
| 355 |
),
|
| 356 |
+
MethodSpec(
|
| 357 |
+
key="residual_k4_composemasked_compbonus_grid035040045_noopbonus003",
|
| 358 |
+
label="K4 composed type-consensus tangents, masked, component no-op bonus 0.03",
|
| 359 |
+
summary_path=(
|
| 360 |
+
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
|
| 361 |
+
"k4_composemasked_compbonus_grid035040045_safe_margin0p20_noopbonus0p03_summary.json"
|
| 362 |
+
),
|
| 363 |
+
),
|
| 364 |
MethodSpec(
|
| 365 |
key="repair_nearmiss_k4_grid025035050_margin020",
|
| 366 |
label="K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20",
|
scripts/build_paper_table_status.py
CHANGED
|
@@ -695,6 +695,16 @@ SPECS = [
|
|
| 695 |
story_role="local tangent composition with anti-goal composite masks on the current best typed prior",
|
| 696 |
pending_job="14911980/14911983",
|
| 697 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 698 |
ResultSpec(
|
| 699 |
key="retrieval_repair_nearmiss_k4_grid025035050_margin020",
|
| 700 |
label="K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20",
|
|
|
|
| 695 |
story_role="local tangent composition with anti-goal composite masks on the current best typed prior",
|
| 696 |
pending_job="14911980/14911983",
|
| 697 |
),
|
| 698 |
+
ResultSpec(
|
| 699 |
+
key="retrieval_residual_k4_composemasked_compbonus_grid035040045_noopbonus003",
|
| 700 |
+
label="K4 composed type-consensus residual retrieval, masked, component no-op bonus 0.03",
|
| 701 |
+
path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_compbonus_grid035040045_safe_margin0p20_noopbonus0p03_summary.json",
|
| 702 |
+
clean_deployment="yes",
|
| 703 |
+
same_state_proposals="no",
|
| 704 |
+
expert_proposal="no",
|
| 705 |
+
story_role="component-wise sparse prior on the masked local tangent composition chart",
|
| 706 |
+
pending_job="14912561/14912562",
|
| 707 |
+
),
|
| 708 |
ResultSpec(
|
| 709 |
key="retrieval_repair_nearmiss_k4_grid025035050_margin020",
|
| 710 |
label="K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20",
|
scripts/eval_maniskill_policy_rollout.py
CHANGED
|
@@ -239,6 +239,12 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 239 |
help="Comma-separated candidate_type=bonus priors added to field potentials before "
|
| 240 |
"selection, e.g. 'residual_no_op=0.05'. Empty preserves previous behavior.",
|
| 241 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 242 |
args = parser.parse_args(argv)
|
| 243 |
lattice_exclude_types = tuple(
|
| 244 |
item.strip() for item in args.lattice_exclude_types.split(",") if item.strip()
|
|
@@ -309,6 +315,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 309 |
retrieval_residual_reduce=args.retrieval_residual_reduce,
|
| 310 |
lattice_exclude_types=lattice_exclude_types,
|
| 311 |
candidate_type_bonuses=candidate_type_bonuses,
|
|
|
|
| 312 |
)
|
| 313 |
print(json.dumps({key: value for key, value in result.items() if key != "rows"}, indent=2))
|
| 314 |
return 0
|
|
|
|
| 239 |
help="Comma-separated candidate_type=bonus priors added to field potentials before "
|
| 240 |
"selection, e.g. 'residual_no_op=0.05'. Empty preserves previous behavior.",
|
| 241 |
)
|
| 242 |
+
parser.add_argument(
|
| 243 |
+
"--candidate-type-bonus-components",
|
| 244 |
+
action="store_true",
|
| 245 |
+
help="Let composite candidate types inherit the sum of configured component bonuses "
|
| 246 |
+
"unless an exact composite bonus is configured.",
|
| 247 |
+
)
|
| 248 |
args = parser.parse_args(argv)
|
| 249 |
lattice_exclude_types = tuple(
|
| 250 |
item.strip() for item in args.lattice_exclude_types.split(",") if item.strip()
|
|
|
|
| 315 |
retrieval_residual_reduce=args.retrieval_residual_reduce,
|
| 316 |
lattice_exclude_types=lattice_exclude_types,
|
| 317 |
candidate_type_bonuses=candidate_type_bonuses,
|
| 318 |
+
candidate_type_bonus_components=args.candidate_type_bonus_components,
|
| 319 |
)
|
| 320 |
print(json.dumps({key: value for key, value in result.items() if key != "rows"}, indent=2))
|
| 321 |
return 0
|
scripts/slurm/eval_maniskill_policy_rollout.sbatch
CHANGED
|
@@ -77,6 +77,7 @@ CANDIDATE_TYPE_BONUSES="${CANDIDATE_TYPE_BONUSES:-}"
|
|
| 77 |
if [[ -n "${CANDIDATE_TYPE_BONUSES_COLON:-}" ]]; then
|
| 78 |
CANDIDATE_TYPE_BONUSES="${CANDIDATE_TYPE_BONUSES_COLON//:/,}"
|
| 79 |
fi
|
|
|
|
| 80 |
|
| 81 |
module load StdEnv/2023 apptainer/1.4.5
|
| 82 |
cd "$PROJECT_DIR"
|
|
@@ -102,6 +103,9 @@ fi
|
|
| 102 |
if [[ "$PREPEND_POLICY_CANDIDATE" == "1" ]]; then
|
| 103 |
EXTRA_ARGS+=(--prepend-policy-candidate)
|
| 104 |
fi
|
|
|
|
|
|
|
|
|
|
| 105 |
|
| 106 |
apptainer exec --nv \
|
| 107 |
--env "LD_LIBRARY_PATH=$CPU_RENDER_LIBS/lib:$NATIVE_LIBS:/.singularity.d/libs,VK_ICD_FILENAMES=$VULKAN_ICD,VK_DRIVER_FILES=$VULKAN_ICD,XDG_RUNTIME_DIR=$RUNTIME_DIR,MESA_SHADER_CACHE_DIR=$CACHE_DIR,LIBGL_ALWAYS_SOFTWARE=1,LP_NUM_THREADS=1,SSL_CERT_FILE=$CA_BUNDLE,REQUESTS_CA_BUNDLE=$CA_BUNDLE,OMP_NUM_THREADS=1,OPENBLAS_NUM_THREADS=1,MKL_NUM_THREADS=1,DOVLA_TORCH_THREADS=1,MPLBACKEND=Agg,PYTHONDONTWRITEBYTECODE=1" \
|
|
|
|
| 77 |
if [[ -n "${CANDIDATE_TYPE_BONUSES_COLON:-}" ]]; then
|
| 78 |
CANDIDATE_TYPE_BONUSES="${CANDIDATE_TYPE_BONUSES_COLON//:/,}"
|
| 79 |
fi
|
| 80 |
+
CANDIDATE_TYPE_BONUS_COMPONENTS="${CANDIDATE_TYPE_BONUS_COMPONENTS:-0}"
|
| 81 |
|
| 82 |
module load StdEnv/2023 apptainer/1.4.5
|
| 83 |
cd "$PROJECT_DIR"
|
|
|
|
| 103 |
if [[ "$PREPEND_POLICY_CANDIDATE" == "1" ]]; then
|
| 104 |
EXTRA_ARGS+=(--prepend-policy-candidate)
|
| 105 |
fi
|
| 106 |
+
if [[ "$CANDIDATE_TYPE_BONUS_COMPONENTS" == "1" ]]; then
|
| 107 |
+
EXTRA_ARGS+=(--candidate-type-bonus-components)
|
| 108 |
+
fi
|
| 109 |
|
| 110 |
apptainer exec --nv \
|
| 111 |
--env "LD_LIBRARY_PATH=$CPU_RENDER_LIBS/lib:$NATIVE_LIBS:/.singularity.d/libs,VK_ICD_FILENAMES=$VULKAN_ICD,VK_DRIVER_FILES=$VULKAN_ICD,XDG_RUNTIME_DIR=$RUNTIME_DIR,MESA_SHADER_CACHE_DIR=$CACHE_DIR,LIBGL_ALWAYS_SOFTWARE=1,LP_NUM_THREADS=1,SSL_CERT_FILE=$CA_BUNDLE,REQUESTS_CA_BUNDLE=$CA_BUNDLE,OMP_NUM_THREADS=1,OPENBLAS_NUM_THREADS=1,MKL_NUM_THREADS=1,DOVLA_TORCH_THREADS=1,MPLBACKEND=Agg,PYTHONDONTWRITEBYTECODE=1" \
|
scripts/slurm/eval_maniskill_policy_rollout_cpu_smoke.sbatch
CHANGED
|
@@ -76,6 +76,7 @@ CANDIDATE_TYPE_BONUSES="${CANDIDATE_TYPE_BONUSES:-}"
|
|
| 76 |
if [[ -n "${CANDIDATE_TYPE_BONUSES_COLON:-}" ]]; then
|
| 77 |
CANDIDATE_TYPE_BONUSES="${CANDIDATE_TYPE_BONUSES_COLON//:/,}"
|
| 78 |
fi
|
|
|
|
| 79 |
|
| 80 |
module load StdEnv/2023 apptainer/1.4.5
|
| 81 |
cd "$PROJECT_DIR"
|
|
@@ -98,6 +99,9 @@ fi
|
|
| 98 |
if [[ "$MAX_GROUPS" != "all" ]]; then
|
| 99 |
EXTRA_ARGS+=(--max-groups "$MAX_GROUPS")
|
| 100 |
fi
|
|
|
|
|
|
|
|
|
|
| 101 |
|
| 102 |
apptainer exec \
|
| 103 |
--env "LD_LIBRARY_PATH=$CPU_RENDER_LIBS/lib:$NATIVE_LIBS,VK_ICD_FILENAMES=$VULKAN_ICD,VK_DRIVER_FILES=$VULKAN_ICD,XDG_RUNTIME_DIR=$RUNTIME_DIR,MESA_SHADER_CACHE_DIR=$CACHE_DIR,LIBGL_ALWAYS_SOFTWARE=1,LP_NUM_THREADS=1,SSL_CERT_FILE=$CA_BUNDLE,REQUESTS_CA_BUNDLE=$CA_BUNDLE,OMP_NUM_THREADS=1,OPENBLAS_NUM_THREADS=1,MKL_NUM_THREADS=1,DOVLA_TORCH_THREADS=1,MPLBACKEND=Agg,PYTHONDONTWRITEBYTECODE=1" \
|
|
|
|
| 76 |
if [[ -n "${CANDIDATE_TYPE_BONUSES_COLON:-}" ]]; then
|
| 77 |
CANDIDATE_TYPE_BONUSES="${CANDIDATE_TYPE_BONUSES_COLON//:/,}"
|
| 78 |
fi
|
| 79 |
+
CANDIDATE_TYPE_BONUS_COMPONENTS="${CANDIDATE_TYPE_BONUS_COMPONENTS:-0}"
|
| 80 |
|
| 81 |
module load StdEnv/2023 apptainer/1.4.5
|
| 82 |
cd "$PROJECT_DIR"
|
|
|
|
| 99 |
if [[ "$MAX_GROUPS" != "all" ]]; then
|
| 100 |
EXTRA_ARGS+=(--max-groups "$MAX_GROUPS")
|
| 101 |
fi
|
| 102 |
+
if [[ "$CANDIDATE_TYPE_BONUS_COMPONENTS" == "1" ]]; then
|
| 103 |
+
EXTRA_ARGS+=(--candidate-type-bonus-components)
|
| 104 |
+
fi
|
| 105 |
|
| 106 |
apptainer exec \
|
| 107 |
--env "LD_LIBRARY_PATH=$CPU_RENDER_LIBS/lib:$NATIVE_LIBS,VK_ICD_FILENAMES=$VULKAN_ICD,VK_DRIVER_FILES=$VULKAN_ICD,XDG_RUNTIME_DIR=$RUNTIME_DIR,MESA_SHADER_CACHE_DIR=$CACHE_DIR,LIBGL_ALWAYS_SOFTWARE=1,LP_NUM_THREADS=1,SSL_CERT_FILE=$CA_BUNDLE,REQUESTS_CA_BUNDLE=$CA_BUNDLE,OMP_NUM_THREADS=1,OPENBLAS_NUM_THREADS=1,MKL_NUM_THREADS=1,DOVLA_TORCH_THREADS=1,MPLBACKEND=Agg,PYTHONDONTWRITEBYTECODE=1" \
|
scripts/slurm/summarize_h16_policy_ckpt.sbatch
CHANGED
|
@@ -99,6 +99,9 @@ for result_path in sorted(base_dir.glob(f"seed_*/{out_name}")):
|
|
| 99 |
),
|
| 100 |
"retrieval_residual_reduce": data.get("retrieval_residual_reduce", "none"),
|
| 101 |
"candidate_type_bonuses": data.get("candidate_type_bonuses", {}),
|
|
|
|
|
|
|
|
|
|
| 102 |
"selected_residual_scale_counts": dict(selected_scale_counts),
|
| 103 |
"policy_rollout_success_rate": data.get("policy_rollout_success_rate", 0.0),
|
| 104 |
"policy_rollout_progress": data.get("policy_rollout_progress", 0.0),
|
|
|
|
| 99 |
),
|
| 100 |
"retrieval_residual_reduce": data.get("retrieval_residual_reduce", "none"),
|
| 101 |
"candidate_type_bonuses": data.get("candidate_type_bonuses", {}),
|
| 102 |
+
"candidate_type_bonus_components": data.get(
|
| 103 |
+
"candidate_type_bonus_components", False
|
| 104 |
+
),
|
| 105 |
"selected_residual_scale_counts": dict(selected_scale_counts),
|
| 106 |
"policy_rollout_success_rate": data.get("policy_rollout_success_rate", 0.0),
|
| 107 |
"policy_rollout_progress": data.get("policy_rollout_progress", 0.0),
|
tests/test_maniskill_policy_rollout.py
CHANGED
|
@@ -20,6 +20,7 @@ from dovla_cil.eval.maniskill_policy_rollout import (
|
|
| 20 |
_attach_retrieved_residual_candidates,
|
| 21 |
_effective_lattice_candidate_count,
|
| 22 |
_lattice_candidate_mask,
|
|
|
|
| 23 |
_load_state_archive,
|
| 24 |
_numeric_action_values,
|
| 25 |
_reduce_residual_candidates_by_type,
|
|
@@ -387,6 +388,60 @@ def test_lattice_candidate_mask_excludes_composite_candidate_parts() -> None:
|
|
| 387 |
assert mask.tolist() == [[True, False, True]]
|
| 388 |
|
| 389 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 390 |
def test_lattice_mode_can_prepend_policy_baseline_for_margin_abstention() -> None:
|
| 391 |
import torch
|
| 392 |
|
|
|
|
| 20 |
_attach_retrieved_residual_candidates,
|
| 21 |
_effective_lattice_candidate_count,
|
| 22 |
_lattice_candidate_mask,
|
| 23 |
+
_lattice_candidate_type_bonus,
|
| 24 |
_load_state_archive,
|
| 25 |
_numeric_action_values,
|
| 26 |
_reduce_residual_candidates_by_type,
|
|
|
|
| 388 |
assert mask.tolist() == [[True, False, True]]
|
| 389 |
|
| 390 |
|
| 391 |
+
def test_lattice_candidate_type_bonus_can_sum_composite_parts() -> None:
|
| 392 |
+
import torch
|
| 393 |
+
|
| 394 |
+
case = _RolloutCase(
|
| 395 |
+
group_id="g",
|
| 396 |
+
task_id="PickCube-v1",
|
| 397 |
+
source_dataset=Path("."),
|
| 398 |
+
state={},
|
| 399 |
+
observation={"features": [0.0]},
|
| 400 |
+
instruction="pick",
|
| 401 |
+
oracle_score=1.0,
|
| 402 |
+
oracle_success=True,
|
| 403 |
+
expert_score=1.0,
|
| 404 |
+
expert_success=True,
|
| 405 |
+
best_action_values=[[0.0]],
|
| 406 |
+
candidate_action_values=[[[0.0]], [[0.2]], [[0.4]]],
|
| 407 |
+
candidate_types=[
|
| 408 |
+
"policy_residual",
|
| 409 |
+
"residual_no_op+residual_wrong_gripper",
|
| 410 |
+
"residual_no_op+residual_random_negative",
|
| 411 |
+
],
|
| 412 |
+
candidate_score_bonuses=[0.0, 0.01, 0.0],
|
| 413 |
+
)
|
| 414 |
+
|
| 415 |
+
exact_only = _lattice_candidate_type_bonus(
|
| 416 |
+
[case],
|
| 417 |
+
torch=torch,
|
| 418 |
+
device="cpu",
|
| 419 |
+
candidate_type_bonuses={"residual_no_op": 0.03, "residual_wrong_gripper": 0.02},
|
| 420 |
+
)
|
| 421 |
+
component = _lattice_candidate_type_bonus(
|
| 422 |
+
[case],
|
| 423 |
+
torch=torch,
|
| 424 |
+
device="cpu",
|
| 425 |
+
candidate_type_bonuses={"residual_no_op": 0.03, "residual_wrong_gripper": 0.02},
|
| 426 |
+
use_components=True,
|
| 427 |
+
)
|
| 428 |
+
exact_override = _lattice_candidate_type_bonus(
|
| 429 |
+
[case],
|
| 430 |
+
torch=torch,
|
| 431 |
+
device="cpu",
|
| 432 |
+
candidate_type_bonuses={
|
| 433 |
+
"residual_no_op": 0.03,
|
| 434 |
+
"residual_wrong_gripper": 0.02,
|
| 435 |
+
"residual_no_op+residual_wrong_gripper": 0.07,
|
| 436 |
+
},
|
| 437 |
+
use_components=True,
|
| 438 |
+
)
|
| 439 |
+
|
| 440 |
+
assert torch.allclose(exact_only, torch.tensor([[0.0, 0.01, 0.0]]))
|
| 441 |
+
assert torch.allclose(component, torch.tensor([[0.0, 0.06, 0.03]]))
|
| 442 |
+
assert torch.allclose(exact_override, torch.tensor([[0.0, 0.08, 0.03]]))
|
| 443 |
+
|
| 444 |
+
|
| 445 |
def test_lattice_mode_can_prepend_policy_baseline_for_margin_abstention() -> None:
|
| 446 |
import torch
|
| 447 |
|