anhtld commited on
Commit
6ea1b3b
·
verified ·
1 Parent(s): 6b00e83

auto-sync 2026-07-02T15:18:17Z workspace (part 4)

Browse files
workspace/results/paper_analysis.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "best_clean_key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005",
3
- "generated_utc": "2026-07-02T14:59:38+00:00",
4
  "mechanism_gap": {
5
  "best_clean_vs_direct_same_ckpt": 0.1060869565217391,
6
  "best_clean_vs_h16": 0.09159420289855075,
@@ -12656,6 +12656,34 @@
12656
  "source": "results/h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_summary.json",
12657
  "std_success": 0.018595959341849776
12658
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12659
  "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_typesuccess001": {
12660
  "ci95_success": 0.04240637681159422,
12661
  "headline_ci95_success": 0.04240637681159422,
 
1
  {
2
  "best_clean_key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005",
3
+ "generated_utc": "2026-07-02T15:05:57+00:00",
4
  "mechanism_gap": {
5
  "best_clean_vs_direct_same_ckpt": 0.1060869565217391,
6
  "best_clean_vs_h16": 0.09159420289855075,
 
12656
  "source": "results/h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_summary.json",
12657
  "std_success": 0.018595959341849776
12658
  },
12659
+ "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025": {
12660
+ "headline_metric": "mean_success",
12661
+ "headline_metric_label": "deployment",
12662
+ "label": "K6-matched transported residual field re-grounding, small train-oracle selector calibration",
12663
+ "missing": true,
12664
+ "source": "results/h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025_summary.json"
12665
+ },
12666
+ "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005": {
12667
+ "headline_metric": "mean_success",
12668
+ "headline_metric_label": "deployment",
12669
+ "label": "K6-matched transported residual field re-grounding, train-oracle calibrated selector",
12670
+ "missing": true,
12671
+ "source": "results/h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005_summary.json"
12672
+ },
12673
+ "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005": {
12674
+ "headline_metric": "mean_success",
12675
+ "headline_metric_label": "deployment",
12676
+ "label": "K6-matched transported residual field re-grounding, train-oracle rank calibration",
12677
+ "missing": true,
12678
+ "source": "results/h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005_summary.json"
12679
+ },
12680
+ "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005": {
12681
+ "headline_metric": "mean_success",
12682
+ "headline_metric_label": "deployment",
12683
+ "label": "K6-matched transported residual field re-grounding, train-oracle type calibration",
12684
+ "missing": true,
12685
+ "source": "results/h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005_summary.json"
12686
+ },
12687
  "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_typesuccess001": {
12688
  "ci95_success": 0.04240637681159422,
12689
  "headline_ci95_success": 0.04240637681159422,
workspace/results/paper_analysis.md CHANGED
@@ -1,6 +1,6 @@
1
  # Paper Analysis
2
 
3
- Generated: `2026-07-02T14:59:38+00:00`
4
 
5
  ## Main Seed Statistics
6
 
@@ -72,6 +72,10 @@ Generated: `2026-07-02T14:59:38+00:00`
72
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrel | K6-matched transported residual field re-grounding, source-score 0.01, task-relative retrieval chart | 3 | deployment | 37.22% +/- 1.66 | 37.22% | +/- 4.12 | 58.49% | 0.512 | +7.48 pp |
73
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrelz | K6-matched transported residual field re-grounding, source-score 0.01, task-relative z-score retrieval chart | 3 | deployment | 36.99% +/- 2.02 | 36.99% | +/- 5.01 | 58.49% | 0.511 | +7.25 pp |
74
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005 | K6-matched transported residual field re-grounding, task-conditioned source-score prior | 3 | deployment | 38.90% +/- 1.86 | 38.90% | +/- 4.62 | 59.96% | 0.513 | +9.16 pp |
 
 
 
 
75
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8 | K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8 | 3 | candidate-oracle | 44.35% +/- 1.98 | 38.84% | +/- 4.91 | 59.95% | 0.512 | +14.61 pp |
76
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0005 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.005 | 3 | deployment | 38.72% +/- 1.71 | 38.72% | +/- 4.24 | 59.90% | 0.511 | +8.99 pp |
77
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0015 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.015 | 3 | deployment | 38.78% +/- 1.82 | 38.78% | +/- 4.51 | 59.94% | 0.511 | +9.04 pp |
 
1
  # Paper Analysis
2
 
3
+ Generated: `2026-07-02T15:05:57+00:00`
4
 
5
  ## Main Seed Statistics
6
 
 
72
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrel | K6-matched transported residual field re-grounding, source-score 0.01, task-relative retrieval chart | 3 | deployment | 37.22% +/- 1.66 | 37.22% | +/- 4.12 | 58.49% | 0.512 | +7.48 pp |
73
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrelz | K6-matched transported residual field re-grounding, source-score 0.01, task-relative z-score retrieval chart | 3 | deployment | 36.99% +/- 2.02 | 36.99% | +/- 5.01 | 58.49% | 0.511 | +7.25 pp |
74
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005 | K6-matched transported residual field re-grounding, task-conditioned source-score prior | 3 | deployment | 38.90% +/- 1.86 | 38.90% | +/- 4.62 | 59.96% | 0.513 | +9.16 pp |
75
+ | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005 | K6-matched transported residual field re-grounding, train-oracle calibrated selector | 0 | deployment | missing | missing | missing | missing | missing | missing |
76
+ | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025 | K6-matched transported residual field re-grounding, small train-oracle selector calibration | 0 | deployment | missing | missing | missing | missing | missing | missing |
77
+ | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005 | K6-matched transported residual field re-grounding, train-oracle rank calibration | 0 | deployment | missing | missing | missing | missing | missing | missing |
78
+ | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005 | K6-matched transported residual field re-grounding, train-oracle type calibration | 0 | deployment | missing | missing | missing | missing | missing | missing |
79
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8 | K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8 | 3 | candidate-oracle | 44.35% +/- 1.98 | 38.84% | +/- 4.91 | 59.95% | 0.512 | +14.61 pp |
80
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0005 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.005 | 3 | deployment | 38.72% +/- 1.71 | 38.72% | +/- 4.24 | 59.90% | 0.511 | +8.99 pp |
81
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0015 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.015 | 3 | deployment | 38.78% +/- 1.82 | 38.78% | +/- 4.51 | 59.94% | 0.511 | +9.04 pp |
workspace/results/paper_table_status.json CHANGED
@@ -1771,6 +1771,82 @@
1771
  "best_config": null,
1772
  "gain_vs_h16_policy": 0.09159420289855075
1773
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1774
  {
1775
  "key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
1776
  "label": "K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8",
 
1771
  "best_config": null,
1772
  "gain_vs_h16_policy": 0.09159420289855075
1773
  },
1774
+ {
1775
+ "key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005",
1776
+ "label": "K6-matched transported residual field re-grounding, train-oracle calibrated selector",
1777
+ "path": "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005_summary.json",
1778
+ "clean_deployment": "yes",
1779
+ "same_state_proposals": "no",
1780
+ "expert_proposal": "no",
1781
+ "story_role": "selector calibration learned from train-split candidate-oracle rollouts and evaluated on held-out validation",
1782
+ "fallback_success": null,
1783
+ "pending_job": "",
1784
+ "path_exists": false,
1785
+ "status": "pending",
1786
+ "success": null,
1787
+ "std_success": null,
1788
+ "completed_seeds": null,
1789
+ "num_completed": null,
1790
+ "best_config": null,
1791
+ "gain_vs_h16_policy": null
1792
+ },
1793
+ {
1794
+ "key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025",
1795
+ "label": "K6-matched transported residual field re-grounding, small train-oracle selector calibration",
1796
+ "path": "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025_summary.json",
1797
+ "clean_deployment": "yes",
1798
+ "same_state_proposals": "no",
1799
+ "expert_proposal": "no",
1800
+ "story_role": "smaller train-oracle rank/type selector calibration learned without validation outcomes",
1801
+ "fallback_success": null,
1802
+ "pending_job": "",
1803
+ "path_exists": false,
1804
+ "status": "pending",
1805
+ "success": null,
1806
+ "std_success": null,
1807
+ "completed_seeds": null,
1808
+ "num_completed": null,
1809
+ "best_config": null,
1810
+ "gain_vs_h16_policy": null
1811
+ },
1812
+ {
1813
+ "key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005",
1814
+ "label": "K6-matched transported residual field re-grounding, train-oracle rank calibration",
1815
+ "path": "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005_summary.json",
1816
+ "clean_deployment": "yes",
1817
+ "same_state_proposals": "no",
1818
+ "expert_proposal": "no",
1819
+ "story_role": "rank-only selector calibration learned from train-split candidate-oracle rollouts",
1820
+ "fallback_success": null,
1821
+ "pending_job": "",
1822
+ "path_exists": false,
1823
+ "status": "pending",
1824
+ "success": null,
1825
+ "std_success": null,
1826
+ "completed_seeds": null,
1827
+ "num_completed": null,
1828
+ "best_config": null,
1829
+ "gain_vs_h16_policy": null
1830
+ },
1831
+ {
1832
+ "key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005",
1833
+ "label": "K6-matched transported residual field re-grounding, train-oracle type calibration",
1834
+ "path": "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005_summary.json",
1835
+ "clean_deployment": "yes",
1836
+ "same_state_proposals": "no",
1837
+ "expert_proposal": "no",
1838
+ "story_role": "type-only selector calibration learned from train-split candidate-oracle rollouts",
1839
+ "fallback_success": null,
1840
+ "pending_job": "",
1841
+ "path_exists": false,
1842
+ "status": "pending",
1843
+ "success": null,
1844
+ "std_success": null,
1845
+ "completed_seeds": null,
1846
+ "num_completed": null,
1847
+ "best_config": null,
1848
+ "gain_vs_h16_policy": null
1849
+ },
1850
  {
1851
  "key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
1852
  "label": "K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8",
workspace/results/paper_table_status.md CHANGED
@@ -96,6 +96,10 @@ Baseline h=16 policy: 29.74%
96
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrel | K6-matched transported residual field re-grounding, source-score 0.01, task-relative retrieval chart | complete | 37.22% | +7.48 pp | yes | no | no | object-relative local chart ablation for transported residual support selection |
97
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrelz | K6-matched transported residual field re-grounding, source-score 0.01, task-relative z-score retrieval chart | complete | 36.99% | +7.25 pp | yes | no | no | normalized object-relative local chart ablation for transported residual support selection |
98
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005 | K6-matched transported residual field re-grounding, task-conditioned source-score prior | complete | 38.90% | +9.16 pp | yes | no | no | task-conditioned train-source utility prior chosen from per-task source-score sensitivity |
 
 
 
 
99
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8 | K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8 | complete | 44.35% | +14.61 pp | diagnostic | no | no | diagnostic top-8 proposal-oracle ceiling for the current best source-score-calibrated selector |
100
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0005 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.005 | complete | 38.72% | +8.99 pp | yes | no | no | very small train-source utility prior sensitivity for selector calibration |
101
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0015 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.015 | complete | 38.78% | +9.04 pp | yes | no | no | midpoint train-source utility prior sensitivity for selector calibration |
 
96
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrel | K6-matched transported residual field re-grounding, source-score 0.01, task-relative retrieval chart | complete | 37.22% | +7.48 pp | yes | no | no | object-relative local chart ablation for transported residual support selection |
97
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrelz | K6-matched transported residual field re-grounding, source-score 0.01, task-relative z-score retrieval chart | complete | 36.99% | +7.25 pp | yes | no | no | normalized object-relative local chart ablation for transported residual support selection |
98
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005 | K6-matched transported residual field re-grounding, task-conditioned source-score prior | complete | 38.90% | +9.16 pp | yes | no | no | task-conditioned train-source utility prior chosen from per-task source-score sensitivity |
99
+ | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005 | K6-matched transported residual field re-grounding, train-oracle calibrated selector | pending | pending | pending | yes | no | no | selector calibration learned from train-split candidate-oracle rollouts and evaluated on held-out validation |
100
+ | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025 | K6-matched transported residual field re-grounding, small train-oracle selector calibration | pending | pending | pending | yes | no | no | smaller train-oracle rank/type selector calibration learned without validation outcomes |
101
+ | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005 | K6-matched transported residual field re-grounding, train-oracle rank calibration | pending | pending | pending | yes | no | no | rank-only selector calibration learned from train-split candidate-oracle rollouts |
102
+ | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005 | K6-matched transported residual field re-grounding, train-oracle type calibration | pending | pending | pending | yes | no | no | type-only selector calibration learned from train-split candidate-oracle rollouts |
103
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8 | K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8 | complete | 44.35% | +14.61 pp | diagnostic | no | no | diagnostic top-8 proposal-oracle ceiling for the current best source-score-calibrated selector |
104
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0005 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.005 | complete | 38.72% | +8.99 pp | yes | no | no | very small train-source utility prior sensitivity for selector calibration |
105
  | transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0015 | K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.015 | complete | 38.78% | +9.04 pp | yes | no | no | midpoint train-source utility prior sensitivity for selector calibration |
workspace/scripts/build_oracle_selector_calibration.py ADDED
@@ -0,0 +1,294 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import glob
6
+ import json
7
+ from collections import defaultdict
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+
12
+ def _mean(values: list[float]) -> float | None:
13
+ if not values:
14
+ return None
15
+ return float(sum(values) / len(values))
16
+
17
+
18
+ def _clip(value: float, limit: float | None) -> float:
19
+ if limit is None or limit <= 0:
20
+ return value
21
+ return max(-float(limit), min(float(limit), value))
22
+
23
+
24
+ def _branch_values(row: dict[str, Any], objective: str) -> list[float]:
25
+ if objective == "score":
26
+ values = row.get("candidate_oracle_branch_scores")
27
+ elif objective == "progress":
28
+ values = row.get("candidate_oracle_branch_progress")
29
+ elif objective == "success":
30
+ values = [
31
+ 1.0 if bool(value) else 0.0
32
+ for value in row.get("candidate_oracle_branch_successes", [])
33
+ ]
34
+ else:
35
+ raise ValueError("objective must be 'score', 'progress', or 'success'")
36
+ if not isinstance(values, list):
37
+ return []
38
+ parsed: list[float] = []
39
+ for value in values:
40
+ try:
41
+ parsed.append(float(value))
42
+ except (TypeError, ValueError):
43
+ parsed.append(0.0)
44
+ return parsed
45
+
46
+
47
+ def _rank_biases(
48
+ values_by_rank: list[list[float]],
49
+ *,
50
+ scale: float,
51
+ min_count: int,
52
+ max_abs_bias: float | None,
53
+ ) -> tuple[list[float], list[float | None], list[int]]:
54
+ means = [_mean(values) for values in values_by_rank]
55
+ counts = [len(values) for values in values_by_rank]
56
+ baseline = means[0] if means and means[0] is not None and counts[0] >= min_count else None
57
+ biases: list[float] = []
58
+ for rank, mean in enumerate(means):
59
+ if rank == 0 or baseline is None or mean is None or counts[rank] < min_count:
60
+ biases.append(0.0)
61
+ else:
62
+ biases.append(_clip(float(scale) * (float(mean) - float(baseline)), max_abs_bias))
63
+ return biases, means, counts
64
+
65
+
66
+ def _type_bonuses(
67
+ values_by_type: dict[str, list[float]],
68
+ *,
69
+ scale: float,
70
+ min_count: int,
71
+ max_abs_bonus: float | None,
72
+ ) -> tuple[dict[str, float], dict[str, float | None], dict[str, int], float | None]:
73
+ pooled = [value for values in values_by_type.values() for value in values]
74
+ baseline = _mean(pooled)
75
+ means = {
76
+ candidate_type: _mean(values)
77
+ for candidate_type, values in sorted(values_by_type.items())
78
+ }
79
+ counts = {
80
+ candidate_type: len(values)
81
+ for candidate_type, values in sorted(values_by_type.items())
82
+ }
83
+ if baseline is None or len(pooled) < min_count:
84
+ return {}, means, counts, baseline
85
+ bonuses: dict[str, float] = {}
86
+ for candidate_type, mean in means.items():
87
+ if mean is None or counts[candidate_type] < min_count:
88
+ continue
89
+ bonuses[candidate_type] = _clip(
90
+ float(scale) * (float(mean) - float(baseline)),
91
+ max_abs_bonus,
92
+ )
93
+ return bonuses, means, counts, baseline
94
+
95
+
96
+ def _iter_rollout_paths(patterns: list[str]) -> list[Path]:
97
+ paths: list[Path] = []
98
+ for pattern in patterns:
99
+ matches = [Path(path) for path in glob.glob(pattern)]
100
+ paths.extend(matches or [Path(pattern)])
101
+ unique = sorted({path.resolve() for path in paths})
102
+ missing = [path for path in unique if not path.exists()]
103
+ if missing:
104
+ raise FileNotFoundError(f"Missing rollout file(s): {missing}")
105
+ return unique
106
+
107
+
108
+ def build_oracle_selector_calibration(
109
+ rollout_paths: list[Path],
110
+ *,
111
+ objective: str,
112
+ max_rank: int,
113
+ rank_scale: float,
114
+ type_scale: float,
115
+ min_count: int,
116
+ max_abs_rank_bias: float | None,
117
+ max_abs_type_bonus: float | None,
118
+ ) -> dict[str, Any]:
119
+ if max_rank <= 0:
120
+ raise ValueError("max_rank must be positive")
121
+ if min_count <= 0:
122
+ raise ValueError("min_count must be positive")
123
+
124
+ rank_by_task: dict[str, list[list[float]]] = defaultdict(
125
+ lambda: [[] for _ in range(max_rank)]
126
+ )
127
+ type_by_task: dict[str, dict[str, list[float]]] = defaultdict(
128
+ lambda: defaultdict(list)
129
+ )
130
+ global_ranks: list[list[float]] = [[] for _ in range(max_rank)]
131
+ global_types: dict[str, list[float]] = defaultdict(list)
132
+ rows_seen = 0
133
+ rows_used = 0
134
+ skipped_branches = 0
135
+
136
+ for path in rollout_paths:
137
+ payload = json.loads(path.read_text())
138
+ for row in payload.get("rows", []):
139
+ if not isinstance(row, dict):
140
+ continue
141
+ rows_seen += 1
142
+ task_id = str(row.get("task_id") or "")
143
+ values = _branch_values(row, objective)
144
+ candidate_types = row.get("candidate_oracle_types")
145
+ valid_mask = row.get("candidate_oracle_valid_mask")
146
+ if not task_id or not values or not isinstance(candidate_types, list):
147
+ continue
148
+ if not isinstance(valid_mask, list):
149
+ valid_mask = [True] * len(values)
150
+ branch_count = min(max_rank, len(values), len(candidate_types), len(valid_mask))
151
+ if branch_count <= 0:
152
+ continue
153
+ rows_used += 1
154
+ for rank in range(branch_count):
155
+ if not bool(valid_mask[rank]):
156
+ skipped_branches += 1
157
+ continue
158
+ value = float(values[rank])
159
+ candidate_type = str(candidate_types[rank])
160
+ rank_by_task[task_id][rank].append(value)
161
+ global_ranks[rank].append(value)
162
+ type_by_task[task_id][candidate_type].append(value)
163
+ global_types[candidate_type].append(value)
164
+
165
+ field_rank_biases_by_task: dict[str, list[float]] = {}
166
+ rank_utility_means_by_task: dict[str, list[float | None]] = {}
167
+ rank_counts_by_task: dict[str, list[int]] = {}
168
+ for task_id in sorted(rank_by_task):
169
+ biases, means, counts = _rank_biases(
170
+ rank_by_task[task_id],
171
+ scale=rank_scale,
172
+ min_count=min_count,
173
+ max_abs_bias=max_abs_rank_bias,
174
+ )
175
+ field_rank_biases_by_task[task_id] = biases
176
+ rank_utility_means_by_task[task_id] = means
177
+ rank_counts_by_task[task_id] = counts
178
+ global_rank_biases, global_rank_means, global_rank_counts = _rank_biases(
179
+ global_ranks,
180
+ scale=rank_scale,
181
+ min_count=min_count,
182
+ max_abs_bias=max_abs_rank_bias,
183
+ )
184
+ field_rank_biases_by_task["*"] = global_rank_biases
185
+ rank_utility_means_by_task["*"] = global_rank_means
186
+ rank_counts_by_task["*"] = global_rank_counts
187
+
188
+ candidate_type_bonuses_by_task: dict[str, dict[str, float]] = {}
189
+ type_utility_means_by_task: dict[str, dict[str, float | None]] = {}
190
+ type_counts_by_task: dict[str, dict[str, int]] = {}
191
+ type_baselines_by_task: dict[str, float | None] = {}
192
+ for task_id in sorted(type_by_task):
193
+ bonuses, means, counts, baseline = _type_bonuses(
194
+ type_by_task[task_id],
195
+ scale=type_scale,
196
+ min_count=min_count,
197
+ max_abs_bonus=max_abs_type_bonus,
198
+ )
199
+ candidate_type_bonuses_by_task[task_id] = bonuses
200
+ type_utility_means_by_task[task_id] = means
201
+ type_counts_by_task[task_id] = counts
202
+ type_baselines_by_task[task_id] = baseline
203
+ global_type_bonuses, global_type_means, global_type_counts, global_type_baseline = (
204
+ _type_bonuses(
205
+ global_types,
206
+ scale=type_scale,
207
+ min_count=min_count,
208
+ max_abs_bonus=max_abs_type_bonus,
209
+ )
210
+ )
211
+ candidate_type_bonuses_by_task["*"] = global_type_bonuses
212
+ type_utility_means_by_task["*"] = global_type_means
213
+ type_counts_by_task["*"] = global_type_counts
214
+ type_baselines_by_task["*"] = global_type_baseline
215
+
216
+ return {
217
+ "source_rollouts": [str(path) for path in rollout_paths],
218
+ "calibration_source": "candidate_oracle_rollout",
219
+ "objective": objective,
220
+ "max_rank": int(max_rank),
221
+ "rank_scale": float(rank_scale),
222
+ "type_scale": float(type_scale),
223
+ "min_count": int(min_count),
224
+ "max_abs_rank_bias": max_abs_rank_bias,
225
+ "max_abs_type_bonus": max_abs_type_bonus,
226
+ "rows_seen": rows_seen,
227
+ "rows_used": rows_used,
228
+ "skipped_branches": skipped_branches,
229
+ "field_rank_biases_by_task": field_rank_biases_by_task,
230
+ "rank_utility_means_by_task": rank_utility_means_by_task,
231
+ "rank_counts_by_task": rank_counts_by_task,
232
+ "candidate_type_bonuses_by_task": candidate_type_bonuses_by_task,
233
+ "type_utility_means_by_task": type_utility_means_by_task,
234
+ "type_counts_by_task": type_counts_by_task,
235
+ "type_baselines_by_task": type_baselines_by_task,
236
+ }
237
+
238
+
239
+ def main(argv: list[str] | None = None) -> int:
240
+ parser = argparse.ArgumentParser(
241
+ description=(
242
+ "Build train-split selector calibration from candidate-oracle rollout traces."
243
+ )
244
+ )
245
+ parser.add_argument("--rollout", action="append", required=True)
246
+ parser.add_argument("--out", type=Path, required=True)
247
+ parser.add_argument("--objective", choices=("score", "progress", "success"), default="score")
248
+ parser.add_argument("--max-rank", type=int, default=4)
249
+ parser.add_argument("--rank-scale", type=float, default=0.05)
250
+ parser.add_argument("--type-scale", type=float, default=0.05)
251
+ parser.add_argument("--min-count", type=int, default=20)
252
+ parser.add_argument("--max-abs-rank-bias", type=float, default=0.02)
253
+ parser.add_argument("--max-abs-type-bonus", type=float, default=0.02)
254
+ args = parser.parse_args(argv)
255
+
256
+ max_abs_rank_bias = args.max_abs_rank_bias if args.max_abs_rank_bias > 0 else None
257
+ max_abs_type_bonus = args.max_abs_type_bonus if args.max_abs_type_bonus > 0 else None
258
+ rollout_paths = _iter_rollout_paths(args.rollout)
259
+ result = build_oracle_selector_calibration(
260
+ rollout_paths,
261
+ objective=args.objective,
262
+ max_rank=args.max_rank,
263
+ rank_scale=args.rank_scale,
264
+ type_scale=args.type_scale,
265
+ min_count=args.min_count,
266
+ max_abs_rank_bias=max_abs_rank_bias,
267
+ max_abs_type_bonus=max_abs_type_bonus,
268
+ )
269
+ args.out.parent.mkdir(parents=True, exist_ok=True)
270
+ args.out.write_text(json.dumps(result, indent=2) + "\n")
271
+ print(
272
+ json.dumps(
273
+ {
274
+ key: value
275
+ for key, value in result.items()
276
+ if key
277
+ not in {
278
+ "field_rank_biases_by_task",
279
+ "rank_utility_means_by_task",
280
+ "rank_counts_by_task",
281
+ "candidate_type_bonuses_by_task",
282
+ "type_utility_means_by_task",
283
+ "type_counts_by_task",
284
+ "type_baselines_by_task",
285
+ }
286
+ },
287
+ indent=2,
288
+ )
289
+ )
290
+ return 0
291
+
292
+
293
+ if __name__ == "__main__":
294
+ raise SystemExit(main())
workspace/scripts/build_paper_analysis.py CHANGED
@@ -561,6 +561,42 @@ METHODS = [
561
  "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_summary.json"
562
  ),
563
  ),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
564
  MethodSpec(
565
  key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
566
  label="K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8",
 
561
  "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_summary.json"
562
  ),
563
  ),
564
+ MethodSpec(
565
+ key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005",
566
+ label="K6-matched transported residual field re-grounding, train-oracle calibrated selector",
567
+ summary_path=(
568
+ "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
569
+ "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
570
+ "oraclecal_train800_k4_r005_t005_summary.json"
571
+ ),
572
+ ),
573
+ MethodSpec(
574
+ key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025",
575
+ label="K6-matched transported residual field re-grounding, small train-oracle selector calibration",
576
+ summary_path=(
577
+ "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
578
+ "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
579
+ "oraclecal_train800_k4_r0025_t0025_summary.json"
580
+ ),
581
+ ),
582
+ MethodSpec(
583
+ key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005",
584
+ label="K6-matched transported residual field re-grounding, train-oracle rank calibration",
585
+ summary_path=(
586
+ "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
587
+ "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
588
+ "oraclecal_train800_k4_rank005_summary.json"
589
+ ),
590
+ ),
591
+ MethodSpec(
592
+ key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005",
593
+ label="K6-matched transported residual field re-grounding, train-oracle type calibration",
594
+ summary_path=(
595
+ "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
596
+ "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
597
+ "oraclecal_train800_k4_type005_summary.json"
598
+ ),
599
+ ),
600
  MethodSpec(
601
  key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
602
  label="K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8",
workspace/scripts/build_paper_table_status.py CHANGED
@@ -927,6 +927,42 @@ SPECS = [
927
  expert_proposal="no",
928
  story_role="task-conditioned train-source utility prior chosen from per-task source-score sensitivity",
929
  ),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
930
  ResultSpec(
931
  key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
932
  label="K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8",
 
927
  expert_proposal="no",
928
  story_role="task-conditioned train-source utility prior chosen from per-task source-score sensitivity",
929
  ),
930
+ ResultSpec(
931
+ key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005",
932
+ label="K6-matched transported residual field re-grounding, train-oracle calibrated selector",
933
+ path="h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005_summary.json",
934
+ clean_deployment="yes",
935
+ same_state_proposals="no",
936
+ expert_proposal="no",
937
+ story_role="selector calibration learned from train-split candidate-oracle rollouts and evaluated on held-out validation",
938
+ ),
939
+ ResultSpec(
940
+ key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025",
941
+ label="K6-matched transported residual field re-grounding, small train-oracle selector calibration",
942
+ path="h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025_summary.json",
943
+ clean_deployment="yes",
944
+ same_state_proposals="no",
945
+ expert_proposal="no",
946
+ story_role="smaller train-oracle rank/type selector calibration learned without validation outcomes",
947
+ ),
948
+ ResultSpec(
949
+ key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005",
950
+ label="K6-matched transported residual field re-grounding, train-oracle rank calibration",
951
+ path="h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005_summary.json",
952
+ clean_deployment="yes",
953
+ same_state_proposals="no",
954
+ expert_proposal="no",
955
+ story_role="rank-only selector calibration learned from train-split candidate-oracle rollouts",
956
+ ),
957
+ ResultSpec(
958
+ key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005",
959
+ label="K6-matched transported residual field re-grounding, train-oracle type calibration",
960
+ path="h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_besttransport_margin0p00_k6_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005_summary.json",
961
+ clean_deployment="yes",
962
+ same_state_proposals="no",
963
+ expert_proposal="no",
964
+ story_role="type-only selector calibration learned from train-split candidate-oracle rollouts",
965
+ ),
966
  ResultSpec(
967
  key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
968
  label="K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8",
workspace/scripts/slurm/build_oracle_selector_calibration.sbatch ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --job-name=build_oracle_cal
3
+ #SBATCH --account=def-yalda
4
+ #SBATCH --time=00:10:00
5
+ #SBATCH --cpus-per-task=1
6
+ #SBATCH --mem=2G
7
+ #SBATCH --array=0-2
8
+ #SBATCH --output=outputs/hpc/logs/%x_%A_%a.out
9
+ #SBATCH --error=outputs/hpc/logs/%x_%A_%a.err
10
+
11
+ set -euo pipefail
12
+
13
+ PROJECT_DIR="${PROJECT_DIR:-$SLURM_SUBMIT_DIR}"
14
+ RUN_ROOT="${RUN_ROOT:-/scratch/$USER/dovla/experiments/dovla_h16_policy_ckpt_runs}"
15
+ OBJECTIVE="${OBJECTIVE:?Set OBJECTIVE to the seed-indexed run objective}"
16
+ SEED="${SLURM_ARRAY_TASK_ID:-0}"
17
+ ROLLOUT_NAME="${ROLLOUT_NAME:?Set ROLLOUT_NAME to the train oracle rollout JSON}"
18
+ OUT_NAME="${OUT_NAME:-oracle_selector_calibration.json}"
19
+ CAL_OBJECTIVE="${CAL_OBJECTIVE:-score}"
20
+ MAX_RANK="${MAX_RANK:-4}"
21
+ RANK_SCALE="${RANK_SCALE:-0.05}"
22
+ TYPE_SCALE="${TYPE_SCALE:-0.05}"
23
+ MIN_COUNT="${MIN_COUNT:-20}"
24
+ MAX_ABS_RANK_BIAS="${MAX_ABS_RANK_BIAS:-0.02}"
25
+ MAX_ABS_TYPE_BONUS="${MAX_ABS_TYPE_BONUS:-0.02}"
26
+ PYTHON="${PYTHON:-python3}"
27
+
28
+ ROLLOUT="$RUN_ROOT/$OBJECTIVE/seed_$SEED/$ROLLOUT_NAME"
29
+ OUT="$RUN_ROOT/$OBJECTIVE/seed_$SEED/$OUT_NAME"
30
+
31
+ cd "$PROJECT_DIR"
32
+ mkdir -p outputs/hpc/logs "$(dirname "$OUT")"
33
+
34
+ "$PYTHON" scripts/build_oracle_selector_calibration.py \
35
+ --rollout "$ROLLOUT" \
36
+ --out "$OUT" \
37
+ --objective "$CAL_OBJECTIVE" \
38
+ --max-rank "$MAX_RANK" \
39
+ --rank-scale "$RANK_SCALE" \
40
+ --type-scale "$TYPE_SCALE" \
41
+ --min-count "$MIN_COUNT" \
42
+ --max-abs-rank-bias "$MAX_ABS_RANK_BIAS" \
43
+ --max-abs-type-bonus "$MAX_ABS_TYPE_BONUS"