auto-sync 2026-07-02T15:18:17Z workspace (part 4)
Browse files- workspace/results/paper_analysis.json +29 -1
- workspace/results/paper_analysis.md +5 -1
- workspace/results/paper_table_status.json +76 -0
- workspace/results/paper_table_status.md +4 -0
- workspace/scripts/build_oracle_selector_calibration.py +294 -0
- workspace/scripts/build_paper_analysis.py +36 -0
- workspace/scripts/build_paper_table_status.py +36 -0
- workspace/scripts/slurm/build_oracle_selector_calibration.sbatch +43 -0
workspace/results/paper_analysis.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"best_clean_key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005",
|
| 3 |
-
"generated_utc": "2026-07-
|
| 4 |
"mechanism_gap": {
|
| 5 |
"best_clean_vs_direct_same_ckpt": 0.1060869565217391,
|
| 6 |
"best_clean_vs_h16": 0.09159420289855075,
|
|
@@ -12656,6 +12656,34 @@
|
|
| 12656 |
"source": "results/h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_summary.json",
|
| 12657 |
"std_success": 0.018595959341849776
|
| 12658 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12659 |
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_typesuccess001": {
|
| 12660 |
"ci95_success": 0.04240637681159422,
|
| 12661 |
"headline_ci95_success": 0.04240637681159422,
|
|
|
|
| 1 |
{
|
| 2 |
"best_clean_key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005",
|
| 3 |
+
"generated_utc": "2026-07-02T15:05:57+00:00",
|
| 4 |
"mechanism_gap": {
|
| 5 |
"best_clean_vs_direct_same_ckpt": 0.1060869565217391,
|
| 6 |
"best_clean_vs_h16": 0.09159420289855075,
|
|
|
|
| 12656 |
"source": "results/h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_summary.json",
|
| 12657 |
"std_success": 0.018595959341849776
|
| 12658 |
},
|
| 12659 |
+
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025": {
|
| 12660 |
+
"headline_metric": "mean_success",
|
| 12661 |
+
"headline_metric_label": "deployment",
|
| 12662 |
+
"label": "K6-matched transported residual field re-grounding, small train-oracle selector calibration",
|
| 12663 |
+
"missing": true,
|
| 12664 |
+
"source": "results/h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025_summary.json"
|
| 12665 |
+
},
|
| 12666 |
+
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005": {
|
| 12667 |
+
"headline_metric": "mean_success",
|
| 12668 |
+
"headline_metric_label": "deployment",
|
| 12669 |
+
"label": "K6-matched transported residual field re-grounding, train-oracle calibrated selector",
|
| 12670 |
+
"missing": true,
|
| 12671 |
+
"source": "results/h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005_summary.json"
|
| 12672 |
+
},
|
| 12673 |
+
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005": {
|
| 12674 |
+
"headline_metric": "mean_success",
|
| 12675 |
+
"headline_metric_label": "deployment",
|
| 12676 |
+
"label": "K6-matched transported residual field re-grounding, train-oracle rank calibration",
|
| 12677 |
+
"missing": true,
|
| 12678 |
+
"source": "results/h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005_summary.json"
|
| 12679 |
+
},
|
| 12680 |
+
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005": {
|
| 12681 |
+
"headline_metric": "mean_success",
|
| 12682 |
+
"headline_metric_label": "deployment",
|
| 12683 |
+
"label": "K6-matched transported residual field re-grounding, train-oracle type calibration",
|
| 12684 |
+
"missing": true,
|
| 12685 |
+
"source": "results/h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005_summary.json"
|
| 12686 |
+
},
|
| 12687 |
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_typesuccess001": {
|
| 12688 |
"ci95_success": 0.04240637681159422,
|
| 12689 |
"headline_ci95_success": 0.04240637681159422,
|
workspace/results/paper_analysis.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
# Paper Analysis
|
| 2 |
|
| 3 |
-
Generated: `2026-07-
|
| 4 |
|
| 5 |
## Main Seed Statistics
|
| 6 |
|
|
@@ -72,6 +72,10 @@ Generated: `2026-07-02T14:59:38+00:00`
|
|
| 72 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrel | K6-matched transported residual field re-grounding, source-score 0.01, task-relative retrieval chart | 3 | deployment | 37.22% +/- 1.66 | 37.22% | +/- 4.12 | 58.49% | 0.512 | +7.48 pp |
|
| 73 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrelz | K6-matched transported residual field re-grounding, source-score 0.01, task-relative z-score retrieval chart | 3 | deployment | 36.99% +/- 2.02 | 36.99% | +/- 5.01 | 58.49% | 0.511 | +7.25 pp |
|
| 74 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005 | K6-matched transported residual field re-grounding, task-conditioned source-score prior | 3 | deployment | 38.90% +/- 1.86 | 38.90% | +/- 4.62 | 59.96% | 0.513 | +9.16 pp |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8 | K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8 | 3 | candidate-oracle | 44.35% +/- 1.98 | 38.84% | +/- 4.91 | 59.95% | 0.512 | +14.61 pp |
|
| 76 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0005 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.005 | 3 | deployment | 38.72% +/- 1.71 | 38.72% | +/- 4.24 | 59.90% | 0.511 | +8.99 pp |
|
| 77 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0015 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.015 | 3 | deployment | 38.78% +/- 1.82 | 38.78% | +/- 4.51 | 59.94% | 0.511 | +9.04 pp |
|
|
|
|
| 1 |
# Paper Analysis
|
| 2 |
|
| 3 |
+
Generated: `2026-07-02T15:05:57+00:00`
|
| 4 |
|
| 5 |
## Main Seed Statistics
|
| 6 |
|
|
|
|
| 72 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrel | K6-matched transported residual field re-grounding, source-score 0.01, task-relative retrieval chart | 3 | deployment | 37.22% +/- 1.66 | 37.22% | +/- 4.12 | 58.49% | 0.512 | +7.48 pp |
|
| 73 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrelz | K6-matched transported residual field re-grounding, source-score 0.01, task-relative z-score retrieval chart | 3 | deployment | 36.99% +/- 2.02 | 36.99% | +/- 5.01 | 58.49% | 0.511 | +7.25 pp |
|
| 74 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005 | K6-matched transported residual field re-grounding, task-conditioned source-score prior | 3 | deployment | 38.90% +/- 1.86 | 38.90% | +/- 4.62 | 59.96% | 0.513 | +9.16 pp |
|
| 75 |
+
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005 | K6-matched transported residual field re-grounding, train-oracle calibrated selector | 0 | deployment | missing | missing | missing | missing | missing | missing |
|
| 76 |
+
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025 | K6-matched transported residual field re-grounding, small train-oracle selector calibration | 0 | deployment | missing | missing | missing | missing | missing | missing |
|
| 77 |
+
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005 | K6-matched transported residual field re-grounding, train-oracle rank calibration | 0 | deployment | missing | missing | missing | missing | missing | missing |
|
| 78 |
+
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005 | K6-matched transported residual field re-grounding, train-oracle type calibration | 0 | deployment | missing | missing | missing | missing | missing | missing |
|
| 79 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8 | K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8 | 3 | candidate-oracle | 44.35% +/- 1.98 | 38.84% | +/- 4.91 | 59.95% | 0.512 | +14.61 pp |
|
| 80 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0005 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.005 | 3 | deployment | 38.72% +/- 1.71 | 38.72% | +/- 4.24 | 59.90% | 0.511 | +8.99 pp |
|
| 81 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0015 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.015 | 3 | deployment | 38.78% +/- 1.82 | 38.78% | +/- 4.51 | 59.94% | 0.511 | +9.04 pp |
|
workspace/results/paper_table_status.json
CHANGED
|
@@ -1771,6 +1771,82 @@
|
|
| 1771 |
"best_config": null,
|
| 1772 |
"gain_vs_h16_policy": 0.09159420289855075
|
| 1773 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1774 |
{
|
| 1775 |
"key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
|
| 1776 |
"label": "K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8",
|
|
|
|
| 1771 |
"best_config": null,
|
| 1772 |
"gain_vs_h16_policy": 0.09159420289855075
|
| 1773 |
},
|
| 1774 |
+
{
|
| 1775 |
+
"key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005",
|
| 1776 |
+
"label": "K6-matched transported residual field re-grounding, train-oracle calibrated selector",
|
| 1777 |
+
"path": "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005_summary.json",
|
| 1778 |
+
"clean_deployment": "yes",
|
| 1779 |
+
"same_state_proposals": "no",
|
| 1780 |
+
"expert_proposal": "no",
|
| 1781 |
+
"story_role": "selector calibration learned from train-split candidate-oracle rollouts and evaluated on held-out validation",
|
| 1782 |
+
"fallback_success": null,
|
| 1783 |
+
"pending_job": "",
|
| 1784 |
+
"path_exists": false,
|
| 1785 |
+
"status": "pending",
|
| 1786 |
+
"success": null,
|
| 1787 |
+
"std_success": null,
|
| 1788 |
+
"completed_seeds": null,
|
| 1789 |
+
"num_completed": null,
|
| 1790 |
+
"best_config": null,
|
| 1791 |
+
"gain_vs_h16_policy": null
|
| 1792 |
+
},
|
| 1793 |
+
{
|
| 1794 |
+
"key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025",
|
| 1795 |
+
"label": "K6-matched transported residual field re-grounding, small train-oracle selector calibration",
|
| 1796 |
+
"path": "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025_summary.json",
|
| 1797 |
+
"clean_deployment": "yes",
|
| 1798 |
+
"same_state_proposals": "no",
|
| 1799 |
+
"expert_proposal": "no",
|
| 1800 |
+
"story_role": "smaller train-oracle rank/type selector calibration learned without validation outcomes",
|
| 1801 |
+
"fallback_success": null,
|
| 1802 |
+
"pending_job": "",
|
| 1803 |
+
"path_exists": false,
|
| 1804 |
+
"status": "pending",
|
| 1805 |
+
"success": null,
|
| 1806 |
+
"std_success": null,
|
| 1807 |
+
"completed_seeds": null,
|
| 1808 |
+
"num_completed": null,
|
| 1809 |
+
"best_config": null,
|
| 1810 |
+
"gain_vs_h16_policy": null
|
| 1811 |
+
},
|
| 1812 |
+
{
|
| 1813 |
+
"key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005",
|
| 1814 |
+
"label": "K6-matched transported residual field re-grounding, train-oracle rank calibration",
|
| 1815 |
+
"path": "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005_summary.json",
|
| 1816 |
+
"clean_deployment": "yes",
|
| 1817 |
+
"same_state_proposals": "no",
|
| 1818 |
+
"expert_proposal": "no",
|
| 1819 |
+
"story_role": "rank-only selector calibration learned from train-split candidate-oracle rollouts",
|
| 1820 |
+
"fallback_success": null,
|
| 1821 |
+
"pending_job": "",
|
| 1822 |
+
"path_exists": false,
|
| 1823 |
+
"status": "pending",
|
| 1824 |
+
"success": null,
|
| 1825 |
+
"std_success": null,
|
| 1826 |
+
"completed_seeds": null,
|
| 1827 |
+
"num_completed": null,
|
| 1828 |
+
"best_config": null,
|
| 1829 |
+
"gain_vs_h16_policy": null
|
| 1830 |
+
},
|
| 1831 |
+
{
|
| 1832 |
+
"key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005",
|
| 1833 |
+
"label": "K6-matched transported residual field re-grounding, train-oracle type calibration",
|
| 1834 |
+
"path": "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005_summary.json",
|
| 1835 |
+
"clean_deployment": "yes",
|
| 1836 |
+
"same_state_proposals": "no",
|
| 1837 |
+
"expert_proposal": "no",
|
| 1838 |
+
"story_role": "type-only selector calibration learned from train-split candidate-oracle rollouts",
|
| 1839 |
+
"fallback_success": null,
|
| 1840 |
+
"pending_job": "",
|
| 1841 |
+
"path_exists": false,
|
| 1842 |
+
"status": "pending",
|
| 1843 |
+
"success": null,
|
| 1844 |
+
"std_success": null,
|
| 1845 |
+
"completed_seeds": null,
|
| 1846 |
+
"num_completed": null,
|
| 1847 |
+
"best_config": null,
|
| 1848 |
+
"gain_vs_h16_policy": null
|
| 1849 |
+
},
|
| 1850 |
{
|
| 1851 |
"key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
|
| 1852 |
"label": "K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8",
|
workspace/results/paper_table_status.md
CHANGED
|
@@ -96,6 +96,10 @@ Baseline h=16 policy: 29.74%
|
|
| 96 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrel | K6-matched transported residual field re-grounding, source-score 0.01, task-relative retrieval chart | complete | 37.22% | +7.48 pp | yes | no | no | object-relative local chart ablation for transported residual support selection |
|
| 97 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrelz | K6-matched transported residual field re-grounding, source-score 0.01, task-relative z-score retrieval chart | complete | 36.99% | +7.25 pp | yes | no | no | normalized object-relative local chart ablation for transported residual support selection |
|
| 98 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005 | K6-matched transported residual field re-grounding, task-conditioned source-score prior | complete | 38.90% | +9.16 pp | yes | no | no | task-conditioned train-source utility prior chosen from per-task source-score sensitivity |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 99 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8 | K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8 | complete | 44.35% | +14.61 pp | diagnostic | no | no | diagnostic top-8 proposal-oracle ceiling for the current best source-score-calibrated selector |
|
| 100 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0005 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.005 | complete | 38.72% | +8.99 pp | yes | no | no | very small train-source utility prior sensitivity for selector calibration |
|
| 101 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0015 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.015 | complete | 38.78% | +9.04 pp | yes | no | no | midpoint train-source utility prior sensitivity for selector calibration |
|
|
|
|
| 96 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrel | K6-matched transported residual field re-grounding, source-score 0.01, task-relative retrieval chart | complete | 37.22% | +7.48 pp | yes | no | no | object-relative local chart ablation for transported residual support selection |
|
| 97 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrelz | K6-matched transported residual field re-grounding, source-score 0.01, task-relative z-score retrieval chart | complete | 36.99% | +7.25 pp | yes | no | no | normalized object-relative local chart ablation for transported residual support selection |
|
| 98 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005 | K6-matched transported residual field re-grounding, task-conditioned source-score prior | complete | 38.90% | +9.16 pp | yes | no | no | task-conditioned train-source utility prior chosen from per-task source-score sensitivity |
|
| 99 |
+
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005 | K6-matched transported residual field re-grounding, train-oracle calibrated selector | pending | pending | pending | yes | no | no | selector calibration learned from train-split candidate-oracle rollouts and evaluated on held-out validation |
|
| 100 |
+
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025 | K6-matched transported residual field re-grounding, small train-oracle selector calibration | pending | pending | pending | yes | no | no | smaller train-oracle rank/type selector calibration learned without validation outcomes |
|
| 101 |
+
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005 | K6-matched transported residual field re-grounding, train-oracle rank calibration | pending | pending | pending | yes | no | no | rank-only selector calibration learned from train-split candidate-oracle rollouts |
|
| 102 |
+
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005 | K6-matched transported residual field re-grounding, train-oracle type calibration | pending | pending | pending | yes | no | no | type-only selector calibration learned from train-split candidate-oracle rollouts |
|
| 103 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8 | K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8 | complete | 44.35% | +14.61 pp | diagnostic | no | no | diagnostic top-8 proposal-oracle ceiling for the current best source-score-calibrated selector |
|
| 104 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0005 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.005 | complete | 38.72% | +8.99 pp | yes | no | no | very small train-source utility prior sensitivity for selector calibration |
|
| 105 |
| transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0015 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.015 | complete | 38.78% | +9.04 pp | yes | no | no | midpoint train-source utility prior sensitivity for selector calibration |
|
workspace/scripts/build_oracle_selector_calibration.py
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import glob
|
| 6 |
+
import json
|
| 7 |
+
from collections import defaultdict
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from typing import Any
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def _mean(values: list[float]) -> float | None:
|
| 13 |
+
if not values:
|
| 14 |
+
return None
|
| 15 |
+
return float(sum(values) / len(values))
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _clip(value: float, limit: float | None) -> float:
|
| 19 |
+
if limit is None or limit <= 0:
|
| 20 |
+
return value
|
| 21 |
+
return max(-float(limit), min(float(limit), value))
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _branch_values(row: dict[str, Any], objective: str) -> list[float]:
|
| 25 |
+
if objective == "score":
|
| 26 |
+
values = row.get("candidate_oracle_branch_scores")
|
| 27 |
+
elif objective == "progress":
|
| 28 |
+
values = row.get("candidate_oracle_branch_progress")
|
| 29 |
+
elif objective == "success":
|
| 30 |
+
values = [
|
| 31 |
+
1.0 if bool(value) else 0.0
|
| 32 |
+
for value in row.get("candidate_oracle_branch_successes", [])
|
| 33 |
+
]
|
| 34 |
+
else:
|
| 35 |
+
raise ValueError("objective must be 'score', 'progress', or 'success'")
|
| 36 |
+
if not isinstance(values, list):
|
| 37 |
+
return []
|
| 38 |
+
parsed: list[float] = []
|
| 39 |
+
for value in values:
|
| 40 |
+
try:
|
| 41 |
+
parsed.append(float(value))
|
| 42 |
+
except (TypeError, ValueError):
|
| 43 |
+
parsed.append(0.0)
|
| 44 |
+
return parsed
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _rank_biases(
|
| 48 |
+
values_by_rank: list[list[float]],
|
| 49 |
+
*,
|
| 50 |
+
scale: float,
|
| 51 |
+
min_count: int,
|
| 52 |
+
max_abs_bias: float | None,
|
| 53 |
+
) -> tuple[list[float], list[float | None], list[int]]:
|
| 54 |
+
means = [_mean(values) for values in values_by_rank]
|
| 55 |
+
counts = [len(values) for values in values_by_rank]
|
| 56 |
+
baseline = means[0] if means and means[0] is not None and counts[0] >= min_count else None
|
| 57 |
+
biases: list[float] = []
|
| 58 |
+
for rank, mean in enumerate(means):
|
| 59 |
+
if rank == 0 or baseline is None or mean is None or counts[rank] < min_count:
|
| 60 |
+
biases.append(0.0)
|
| 61 |
+
else:
|
| 62 |
+
biases.append(_clip(float(scale) * (float(mean) - float(baseline)), max_abs_bias))
|
| 63 |
+
return biases, means, counts
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _type_bonuses(
|
| 67 |
+
values_by_type: dict[str, list[float]],
|
| 68 |
+
*,
|
| 69 |
+
scale: float,
|
| 70 |
+
min_count: int,
|
| 71 |
+
max_abs_bonus: float | None,
|
| 72 |
+
) -> tuple[dict[str, float], dict[str, float | None], dict[str, int], float | None]:
|
| 73 |
+
pooled = [value for values in values_by_type.values() for value in values]
|
| 74 |
+
baseline = _mean(pooled)
|
| 75 |
+
means = {
|
| 76 |
+
candidate_type: _mean(values)
|
| 77 |
+
for candidate_type, values in sorted(values_by_type.items())
|
| 78 |
+
}
|
| 79 |
+
counts = {
|
| 80 |
+
candidate_type: len(values)
|
| 81 |
+
for candidate_type, values in sorted(values_by_type.items())
|
| 82 |
+
}
|
| 83 |
+
if baseline is None or len(pooled) < min_count:
|
| 84 |
+
return {}, means, counts, baseline
|
| 85 |
+
bonuses: dict[str, float] = {}
|
| 86 |
+
for candidate_type, mean in means.items():
|
| 87 |
+
if mean is None or counts[candidate_type] < min_count:
|
| 88 |
+
continue
|
| 89 |
+
bonuses[candidate_type] = _clip(
|
| 90 |
+
float(scale) * (float(mean) - float(baseline)),
|
| 91 |
+
max_abs_bonus,
|
| 92 |
+
)
|
| 93 |
+
return bonuses, means, counts, baseline
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def _iter_rollout_paths(patterns: list[str]) -> list[Path]:
|
| 97 |
+
paths: list[Path] = []
|
| 98 |
+
for pattern in patterns:
|
| 99 |
+
matches = [Path(path) for path in glob.glob(pattern)]
|
| 100 |
+
paths.extend(matches or [Path(pattern)])
|
| 101 |
+
unique = sorted({path.resolve() for path in paths})
|
| 102 |
+
missing = [path for path in unique if not path.exists()]
|
| 103 |
+
if missing:
|
| 104 |
+
raise FileNotFoundError(f"Missing rollout file(s): {missing}")
|
| 105 |
+
return unique
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def build_oracle_selector_calibration(
|
| 109 |
+
rollout_paths: list[Path],
|
| 110 |
+
*,
|
| 111 |
+
objective: str,
|
| 112 |
+
max_rank: int,
|
| 113 |
+
rank_scale: float,
|
| 114 |
+
type_scale: float,
|
| 115 |
+
min_count: int,
|
| 116 |
+
max_abs_rank_bias: float | None,
|
| 117 |
+
max_abs_type_bonus: float | None,
|
| 118 |
+
) -> dict[str, Any]:
|
| 119 |
+
if max_rank <= 0:
|
| 120 |
+
raise ValueError("max_rank must be positive")
|
| 121 |
+
if min_count <= 0:
|
| 122 |
+
raise ValueError("min_count must be positive")
|
| 123 |
+
|
| 124 |
+
rank_by_task: dict[str, list[list[float]]] = defaultdict(
|
| 125 |
+
lambda: [[] for _ in range(max_rank)]
|
| 126 |
+
)
|
| 127 |
+
type_by_task: dict[str, dict[str, list[float]]] = defaultdict(
|
| 128 |
+
lambda: defaultdict(list)
|
| 129 |
+
)
|
| 130 |
+
global_ranks: list[list[float]] = [[] for _ in range(max_rank)]
|
| 131 |
+
global_types: dict[str, list[float]] = defaultdict(list)
|
| 132 |
+
rows_seen = 0
|
| 133 |
+
rows_used = 0
|
| 134 |
+
skipped_branches = 0
|
| 135 |
+
|
| 136 |
+
for path in rollout_paths:
|
| 137 |
+
payload = json.loads(path.read_text())
|
| 138 |
+
for row in payload.get("rows", []):
|
| 139 |
+
if not isinstance(row, dict):
|
| 140 |
+
continue
|
| 141 |
+
rows_seen += 1
|
| 142 |
+
task_id = str(row.get("task_id") or "")
|
| 143 |
+
values = _branch_values(row, objective)
|
| 144 |
+
candidate_types = row.get("candidate_oracle_types")
|
| 145 |
+
valid_mask = row.get("candidate_oracle_valid_mask")
|
| 146 |
+
if not task_id or not values or not isinstance(candidate_types, list):
|
| 147 |
+
continue
|
| 148 |
+
if not isinstance(valid_mask, list):
|
| 149 |
+
valid_mask = [True] * len(values)
|
| 150 |
+
branch_count = min(max_rank, len(values), len(candidate_types), len(valid_mask))
|
| 151 |
+
if branch_count <= 0:
|
| 152 |
+
continue
|
| 153 |
+
rows_used += 1
|
| 154 |
+
for rank in range(branch_count):
|
| 155 |
+
if not bool(valid_mask[rank]):
|
| 156 |
+
skipped_branches += 1
|
| 157 |
+
continue
|
| 158 |
+
value = float(values[rank])
|
| 159 |
+
candidate_type = str(candidate_types[rank])
|
| 160 |
+
rank_by_task[task_id][rank].append(value)
|
| 161 |
+
global_ranks[rank].append(value)
|
| 162 |
+
type_by_task[task_id][candidate_type].append(value)
|
| 163 |
+
global_types[candidate_type].append(value)
|
| 164 |
+
|
| 165 |
+
field_rank_biases_by_task: dict[str, list[float]] = {}
|
| 166 |
+
rank_utility_means_by_task: dict[str, list[float | None]] = {}
|
| 167 |
+
rank_counts_by_task: dict[str, list[int]] = {}
|
| 168 |
+
for task_id in sorted(rank_by_task):
|
| 169 |
+
biases, means, counts = _rank_biases(
|
| 170 |
+
rank_by_task[task_id],
|
| 171 |
+
scale=rank_scale,
|
| 172 |
+
min_count=min_count,
|
| 173 |
+
max_abs_bias=max_abs_rank_bias,
|
| 174 |
+
)
|
| 175 |
+
field_rank_biases_by_task[task_id] = biases
|
| 176 |
+
rank_utility_means_by_task[task_id] = means
|
| 177 |
+
rank_counts_by_task[task_id] = counts
|
| 178 |
+
global_rank_biases, global_rank_means, global_rank_counts = _rank_biases(
|
| 179 |
+
global_ranks,
|
| 180 |
+
scale=rank_scale,
|
| 181 |
+
min_count=min_count,
|
| 182 |
+
max_abs_bias=max_abs_rank_bias,
|
| 183 |
+
)
|
| 184 |
+
field_rank_biases_by_task["*"] = global_rank_biases
|
| 185 |
+
rank_utility_means_by_task["*"] = global_rank_means
|
| 186 |
+
rank_counts_by_task["*"] = global_rank_counts
|
| 187 |
+
|
| 188 |
+
candidate_type_bonuses_by_task: dict[str, dict[str, float]] = {}
|
| 189 |
+
type_utility_means_by_task: dict[str, dict[str, float | None]] = {}
|
| 190 |
+
type_counts_by_task: dict[str, dict[str, int]] = {}
|
| 191 |
+
type_baselines_by_task: dict[str, float | None] = {}
|
| 192 |
+
for task_id in sorted(type_by_task):
|
| 193 |
+
bonuses, means, counts, baseline = _type_bonuses(
|
| 194 |
+
type_by_task[task_id],
|
| 195 |
+
scale=type_scale,
|
| 196 |
+
min_count=min_count,
|
| 197 |
+
max_abs_bonus=max_abs_type_bonus,
|
| 198 |
+
)
|
| 199 |
+
candidate_type_bonuses_by_task[task_id] = bonuses
|
| 200 |
+
type_utility_means_by_task[task_id] = means
|
| 201 |
+
type_counts_by_task[task_id] = counts
|
| 202 |
+
type_baselines_by_task[task_id] = baseline
|
| 203 |
+
global_type_bonuses, global_type_means, global_type_counts, global_type_baseline = (
|
| 204 |
+
_type_bonuses(
|
| 205 |
+
global_types,
|
| 206 |
+
scale=type_scale,
|
| 207 |
+
min_count=min_count,
|
| 208 |
+
max_abs_bonus=max_abs_type_bonus,
|
| 209 |
+
)
|
| 210 |
+
)
|
| 211 |
+
candidate_type_bonuses_by_task["*"] = global_type_bonuses
|
| 212 |
+
type_utility_means_by_task["*"] = global_type_means
|
| 213 |
+
type_counts_by_task["*"] = global_type_counts
|
| 214 |
+
type_baselines_by_task["*"] = global_type_baseline
|
| 215 |
+
|
| 216 |
+
return {
|
| 217 |
+
"source_rollouts": [str(path) for path in rollout_paths],
|
| 218 |
+
"calibration_source": "candidate_oracle_rollout",
|
| 219 |
+
"objective": objective,
|
| 220 |
+
"max_rank": int(max_rank),
|
| 221 |
+
"rank_scale": float(rank_scale),
|
| 222 |
+
"type_scale": float(type_scale),
|
| 223 |
+
"min_count": int(min_count),
|
| 224 |
+
"max_abs_rank_bias": max_abs_rank_bias,
|
| 225 |
+
"max_abs_type_bonus": max_abs_type_bonus,
|
| 226 |
+
"rows_seen": rows_seen,
|
| 227 |
+
"rows_used": rows_used,
|
| 228 |
+
"skipped_branches": skipped_branches,
|
| 229 |
+
"field_rank_biases_by_task": field_rank_biases_by_task,
|
| 230 |
+
"rank_utility_means_by_task": rank_utility_means_by_task,
|
| 231 |
+
"rank_counts_by_task": rank_counts_by_task,
|
| 232 |
+
"candidate_type_bonuses_by_task": candidate_type_bonuses_by_task,
|
| 233 |
+
"type_utility_means_by_task": type_utility_means_by_task,
|
| 234 |
+
"type_counts_by_task": type_counts_by_task,
|
| 235 |
+
"type_baselines_by_task": type_baselines_by_task,
|
| 236 |
+
}
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def main(argv: list[str] | None = None) -> int:
|
| 240 |
+
parser = argparse.ArgumentParser(
|
| 241 |
+
description=(
|
| 242 |
+
"Build train-split selector calibration from candidate-oracle rollout traces."
|
| 243 |
+
)
|
| 244 |
+
)
|
| 245 |
+
parser.add_argument("--rollout", action="append", required=True)
|
| 246 |
+
parser.add_argument("--out", type=Path, required=True)
|
| 247 |
+
parser.add_argument("--objective", choices=("score", "progress", "success"), default="score")
|
| 248 |
+
parser.add_argument("--max-rank", type=int, default=4)
|
| 249 |
+
parser.add_argument("--rank-scale", type=float, default=0.05)
|
| 250 |
+
parser.add_argument("--type-scale", type=float, default=0.05)
|
| 251 |
+
parser.add_argument("--min-count", type=int, default=20)
|
| 252 |
+
parser.add_argument("--max-abs-rank-bias", type=float, default=0.02)
|
| 253 |
+
parser.add_argument("--max-abs-type-bonus", type=float, default=0.02)
|
| 254 |
+
args = parser.parse_args(argv)
|
| 255 |
+
|
| 256 |
+
max_abs_rank_bias = args.max_abs_rank_bias if args.max_abs_rank_bias > 0 else None
|
| 257 |
+
max_abs_type_bonus = args.max_abs_type_bonus if args.max_abs_type_bonus > 0 else None
|
| 258 |
+
rollout_paths = _iter_rollout_paths(args.rollout)
|
| 259 |
+
result = build_oracle_selector_calibration(
|
| 260 |
+
rollout_paths,
|
| 261 |
+
objective=args.objective,
|
| 262 |
+
max_rank=args.max_rank,
|
| 263 |
+
rank_scale=args.rank_scale,
|
| 264 |
+
type_scale=args.type_scale,
|
| 265 |
+
min_count=args.min_count,
|
| 266 |
+
max_abs_rank_bias=max_abs_rank_bias,
|
| 267 |
+
max_abs_type_bonus=max_abs_type_bonus,
|
| 268 |
+
)
|
| 269 |
+
args.out.parent.mkdir(parents=True, exist_ok=True)
|
| 270 |
+
args.out.write_text(json.dumps(result, indent=2) + "\n")
|
| 271 |
+
print(
|
| 272 |
+
json.dumps(
|
| 273 |
+
{
|
| 274 |
+
key: value
|
| 275 |
+
for key, value in result.items()
|
| 276 |
+
if key
|
| 277 |
+
not in {
|
| 278 |
+
"field_rank_biases_by_task",
|
| 279 |
+
"rank_utility_means_by_task",
|
| 280 |
+
"rank_counts_by_task",
|
| 281 |
+
"candidate_type_bonuses_by_task",
|
| 282 |
+
"type_utility_means_by_task",
|
| 283 |
+
"type_counts_by_task",
|
| 284 |
+
"type_baselines_by_task",
|
| 285 |
+
}
|
| 286 |
+
},
|
| 287 |
+
indent=2,
|
| 288 |
+
)
|
| 289 |
+
)
|
| 290 |
+
return 0
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
if __name__ == "__main__":
|
| 294 |
+
raise SystemExit(main())
|
workspace/scripts/build_paper_analysis.py
CHANGED
|
@@ -561,6 +561,42 @@ METHODS = [
|
|
| 561 |
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_summary.json"
|
| 562 |
),
|
| 563 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 564 |
MethodSpec(
|
| 565 |
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
|
| 566 |
label="K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8",
|
|
|
|
| 561 |
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_summary.json"
|
| 562 |
),
|
| 563 |
),
|
| 564 |
+
MethodSpec(
|
| 565 |
+
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005",
|
| 566 |
+
label="K6-matched transported residual field re-grounding, train-oracle calibrated selector",
|
| 567 |
+
summary_path=(
|
| 568 |
+
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
|
| 569 |
+
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
|
| 570 |
+
"oraclecal_train800_k4_r005_t005_summary.json"
|
| 571 |
+
),
|
| 572 |
+
),
|
| 573 |
+
MethodSpec(
|
| 574 |
+
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025",
|
| 575 |
+
label="K6-matched transported residual field re-grounding, small train-oracle selector calibration",
|
| 576 |
+
summary_path=(
|
| 577 |
+
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
|
| 578 |
+
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
|
| 579 |
+
"oraclecal_train800_k4_r0025_t0025_summary.json"
|
| 580 |
+
),
|
| 581 |
+
),
|
| 582 |
+
MethodSpec(
|
| 583 |
+
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005",
|
| 584 |
+
label="K6-matched transported residual field re-grounding, train-oracle rank calibration",
|
| 585 |
+
summary_path=(
|
| 586 |
+
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
|
| 587 |
+
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
|
| 588 |
+
"oraclecal_train800_k4_rank005_summary.json"
|
| 589 |
+
),
|
| 590 |
+
),
|
| 591 |
+
MethodSpec(
|
| 592 |
+
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005",
|
| 593 |
+
label="K6-matched transported residual field re-grounding, train-oracle type calibration",
|
| 594 |
+
summary_path=(
|
| 595 |
+
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
|
| 596 |
+
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
|
| 597 |
+
"oraclecal_train800_k4_type005_summary.json"
|
| 598 |
+
),
|
| 599 |
+
),
|
| 600 |
MethodSpec(
|
| 601 |
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
|
| 602 |
label="K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8",
|
workspace/scripts/build_paper_table_status.py
CHANGED
|
@@ -927,6 +927,42 @@ SPECS = [
|
|
| 927 |
expert_proposal="no",
|
| 928 |
story_role="task-conditioned train-source utility prior chosen from per-task source-score sensitivity",
|
| 929 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 930 |
ResultSpec(
|
| 931 |
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
|
| 932 |
label="K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8",
|
|
|
|
| 927 |
expert_proposal="no",
|
| 928 |
story_role="task-conditioned train-source utility prior chosen from per-task source-score sensitivity",
|
| 929 |
),
|
| 930 |
+
ResultSpec(
|
| 931 |
+
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005",
|
| 932 |
+
label="K6-matched transported residual field re-grounding, train-oracle calibrated selector",
|
| 933 |
+
path="h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005_summary.json",
|
| 934 |
+
clean_deployment="yes",
|
| 935 |
+
same_state_proposals="no",
|
| 936 |
+
expert_proposal="no",
|
| 937 |
+
story_role="selector calibration learned from train-split candidate-oracle rollouts and evaluated on held-out validation",
|
| 938 |
+
),
|
| 939 |
+
ResultSpec(
|
| 940 |
+
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025",
|
| 941 |
+
label="K6-matched transported residual field re-grounding, small train-oracle selector calibration",
|
| 942 |
+
path="h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025_summary.json",
|
| 943 |
+
clean_deployment="yes",
|
| 944 |
+
same_state_proposals="no",
|
| 945 |
+
expert_proposal="no",
|
| 946 |
+
story_role="smaller train-oracle rank/type selector calibration learned without validation outcomes",
|
| 947 |
+
),
|
| 948 |
+
ResultSpec(
|
| 949 |
+
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005",
|
| 950 |
+
label="K6-matched transported residual field re-grounding, train-oracle rank calibration",
|
| 951 |
+
path="h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005_summary.json",
|
| 952 |
+
clean_deployment="yes",
|
| 953 |
+
same_state_proposals="no",
|
| 954 |
+
expert_proposal="no",
|
| 955 |
+
story_role="rank-only selector calibration learned from train-split candidate-oracle rollouts",
|
| 956 |
+
),
|
| 957 |
+
ResultSpec(
|
| 958 |
+
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005",
|
| 959 |
+
label="K6-matched transported residual field re-grounding, train-oracle type calibration",
|
| 960 |
+
path="h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005_summary.json",
|
| 961 |
+
clean_deployment="yes",
|
| 962 |
+
same_state_proposals="no",
|
| 963 |
+
expert_proposal="no",
|
| 964 |
+
story_role="type-only selector calibration learned from train-split candidate-oracle rollouts",
|
| 965 |
+
),
|
| 966 |
ResultSpec(
|
| 967 |
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
|
| 968 |
label="K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8",
|
workspace/scripts/slurm/build_oracle_selector_calibration.sbatch
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
#SBATCH --job-name=build_oracle_cal
|
| 3 |
+
#SBATCH --account=def-yalda
|
| 4 |
+
#SBATCH --time=00:10:00
|
| 5 |
+
#SBATCH --cpus-per-task=1
|
| 6 |
+
#SBATCH --mem=2G
|
| 7 |
+
#SBATCH --array=0-2
|
| 8 |
+
#SBATCH --output=outputs/hpc/logs/%x_%A_%a.out
|
| 9 |
+
#SBATCH --error=outputs/hpc/logs/%x_%A_%a.err
|
| 10 |
+
|
| 11 |
+
set -euo pipefail
|
| 12 |
+
|
| 13 |
+
PROJECT_DIR="${PROJECT_DIR:-$SLURM_SUBMIT_DIR}"
|
| 14 |
+
RUN_ROOT="${RUN_ROOT:-/scratch/$USER/dovla/experiments/dovla_h16_policy_ckpt_runs}"
|
| 15 |
+
OBJECTIVE="${OBJECTIVE:?Set OBJECTIVE to the seed-indexed run objective}"
|
| 16 |
+
SEED="${SLURM_ARRAY_TASK_ID:-0}"
|
| 17 |
+
ROLLOUT_NAME="${ROLLOUT_NAME:?Set ROLLOUT_NAME to the train oracle rollout JSON}"
|
| 18 |
+
OUT_NAME="${OUT_NAME:-oracle_selector_calibration.json}"
|
| 19 |
+
CAL_OBJECTIVE="${CAL_OBJECTIVE:-score}"
|
| 20 |
+
MAX_RANK="${MAX_RANK:-4}"
|
| 21 |
+
RANK_SCALE="${RANK_SCALE:-0.05}"
|
| 22 |
+
TYPE_SCALE="${TYPE_SCALE:-0.05}"
|
| 23 |
+
MIN_COUNT="${MIN_COUNT:-20}"
|
| 24 |
+
MAX_ABS_RANK_BIAS="${MAX_ABS_RANK_BIAS:-0.02}"
|
| 25 |
+
MAX_ABS_TYPE_BONUS="${MAX_ABS_TYPE_BONUS:-0.02}"
|
| 26 |
+
PYTHON="${PYTHON:-python3}"
|
| 27 |
+
|
| 28 |
+
ROLLOUT="$RUN_ROOT/$OBJECTIVE/seed_$SEED/$ROLLOUT_NAME"
|
| 29 |
+
OUT="$RUN_ROOT/$OBJECTIVE/seed_$SEED/$OUT_NAME"
|
| 30 |
+
|
| 31 |
+
cd "$PROJECT_DIR"
|
| 32 |
+
mkdir -p outputs/hpc/logs "$(dirname "$OUT")"
|
| 33 |
+
|
| 34 |
+
"$PYTHON" scripts/build_oracle_selector_calibration.py \
|
| 35 |
+
--rollout "$ROLLOUT" \
|
| 36 |
+
--out "$OUT" \
|
| 37 |
+
--objective "$CAL_OBJECTIVE" \
|
| 38 |
+
--max-rank "$MAX_RANK" \
|
| 39 |
+
--rank-scale "$RANK_SCALE" \
|
| 40 |
+
--type-scale "$TYPE_SCALE" \
|
| 41 |
+
--min-count "$MIN_COUNT" \
|
| 42 |
+
--max-abs-rank-bias "$MAX_ABS_RANK_BIAS" \
|
| 43 |
+
--max-abs-type-bonus "$MAX_ABS_TYPE_BONUS"
|