Upload per-dim trainmax train seed2 artifact

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2/command.txt

@@ -0,0 +1 @@
+python scripts/eval_ctt_generated_rollout.py --checkpoint runs/ctt_residual_base_context_obs_seed2/model.pt --source-index data/cil_charts_rgb_refs/train/index.json --target-index data/cil_charts_rgb_refs/train/index.json --out-dir runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2 --k 8 --pool-size 0 --neighbors 8 --max-target-charts 144 --group-batch-size 1 --device cpu --sim-backend physx_cpu --render-backend cpu --restore-tolerance 1e-5 --delta-scale 1.0 --execution-action-scale 1.0 --execution-action-scale-vector 0.381401261606,0.371117178171,0.354308917074,0.237850081835,0.248271283428,0.232648812617,0.215420039658 --execution-action-transform identity --bootstrap-samples 200 --exclude-self-source --disable-action-clipping

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2/config.yaml

@@ -0,0 +1,22 @@
+bootstrap_samples: 200
+checkpoint: runs/ctt_residual_base_context_obs_seed2/model.pt
+delta_scale: 1.0
+device: cpu
+disable_action_clipping: True
+exclude_self_source: True
+execution_action_scale: 1.0
+execution_action_scale_vector: 0.381401261606,0.371117178171,0.354308917074,0.237850081835,0.248271283428,0.232648812617,0.215420039658
+execution_action_transform: identity
+group_batch_size: 1
+include_targets_without_positives: False
+k: 8
+max_target_charts: 144
+neighbors: 8
+out_dir: runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2
+pool_size: 0
+render_backend: cpu
+restore_tolerance: 1e-05
+sim_backend: physx_cpu
+skip_metrics: False
+source_index: data/cil_charts_rgb_refs/train/index.json
+target_index: data/cil_charts_rgb_refs/train/index.json

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2/data_hash.txt

@@ -0,0 +1 @@
+67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2/git_hash.txt

@@ -0,0 +1 @@
+

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2/measured_metrics/metrics_by_task.json

@@ -0,0 +1,182 @@
+{
+  "LiftPegUpright-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.2,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.7833942204713822,
+    "branch_car_at_8": 0.25610943883657455,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 1.0,
+    "hidden_chart_oracle_utility_at_8": 2.0,
+    "ncar_to_hidden_chart_oracle_at_8": 0.8673945494371688,
+    "ncar_to_proposal_oracle_at_8": 0.22809248414000916,
+    "outcome_ptr_at_8": 0.4,
+    "pairwise_causal_calibration_ece": 0.18048328241453365,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.4,
+    "proposal_oracle_success_gain_over_base_at_8": 0.2,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 1.0454350337386131,
+    "proposal_oracle_utility_gain_over_base_at_8": 0.262040813267231,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.2,
+    "selected_success_gain_over_base_at_8": 0.0,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.7893255949020386,
+    "selected_utility_gain_over_base_at_8": 0.005931374430656433,
+    "selector_gap_fraction_to_hidden_at_8": 0.2500967471724982,
+    "selector_regret_at_8": 0.25610943883657455,
+    "success_selector_gap_at_8": 0.2,
+    "success_support_gap_at_8": 0.6,
+    "success_total_car_to_hidden_at_8": 0.8,
+    "support_gap_at_8": 0.9545649662613869,
+    "support_gap_fraction_to_hidden_at_8": 0.617297807934756,
+    "total_car_to_hidden_at_8": 1.2106744050979614
+  },
+  "PickCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.14285714285714285,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.353121066732066,
+    "branch_car_at_8": 0.21211611798831395,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 1.0,
+    "hidden_chart_oracle_utility_at_8": 2.0,
+    "ncar_to_hidden_chart_oracle_at_8": 1.0109060531044503,
+    "ncar_to_proposal_oracle_at_8": 0.52622527077833,
+    "outcome_ptr_at_8": 0.0,
+    "pairwise_causal_calibration_ece": 0.14872269547853015,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.14285714285714285,
+    "proposal_oracle_success_gain_over_base_at_8": 0.0,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 0.341122750725065,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.01199831600700106,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.0,
+    "selected_success_gain_over_base_at_8": -0.14285714285714285,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.129006632736751,
+    "selected_utility_gain_over_base_at_8": -0.224114433995315,
+    "selector_gap_fraction_to_hidden_at_8": 0.003587880851496335,
+    "selector_regret_at_8": 0.21211611798831395,
+    "success_selector_gap_at_8": 0.14285714285714285,
+    "success_support_gap_at_8": 0.8571428571428571,
+    "success_total_car_to_hidden_at_8": 1.0,
+    "support_gap_at_8": 1.658877249274935,
+    "support_gap_fraction_to_hidden_at_8": 1.0073181775152928,
+    "total_car_to_hidden_at_8": 1.870993367263249
+  },
+  "PullCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.625,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.16666666666666666,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.4476979454047978,
+    "branch_car_at_8": 0.07172732797334902,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.4947916666666667,
+    "hidden_chart_oracle_success_at_8": 0.3888888888888889,
+    "hidden_chart_oracle_utility_at_8": 0.9149795592659049,
+    "ncar_to_hidden_chart_oracle_at_8": 3.144587613695753,
+    "ncar_to_proposal_oracle_at_8": 0.755293067551312,
+    "outcome_ptr_at_8": 0.3055555555555556,
+    "pairwise_causal_calibration_ece": 0.23026700740431424,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.1111111111111111,
+    "proposal_oracle_success_gain_over_base_at_8": -0.05555555555555555,
+    "proposal_oracle_unsafe_known_at_8": 0.4861111111111111,
+    "proposal_oracle_utility_at_8": 0.3342518941693318,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.11344605123546596,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.08333333333333333,
+    "selected_success_gain_over_base_at_8": -0.08333333333333333,
+    "selected_unsafe_known_at_8": 0.5138888888888888,
+    "selected_utility_at_8": 0.2625245661959828,
+    "selected_utility_gain_over_base_at_8": -0.18517337920881496,
+    "selector_gap_fraction_to_hidden_at_8": 0.8563952330010932,
+    "selector_regret_at_8": 0.07172732797334902,
+    "success_selector_gap_at_8": 0.027777777777777776,
+    "success_support_gap_at_8": 0.2777777777777778,
+    "success_total_car_to_hidden_at_8": 0.3055555555555556,
+    "support_gap_at_8": 0.5844958343142126,
+    "support_gap_fraction_to_hidden_at_8": 2.3021517980465727,
+    "total_car_to_hidden_at_8": 0.6557380551918565
+  },
+  "PushCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.7857142857142857,
+    "base_unsafe_known": 0.0,
+    "base_utility": 1.588915357100112,
+    "branch_car_at_8": 0.4667696101324899,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 1.0,
+    "hidden_chart_oracle_utility_at_8": 2.0,
+    "ncar_to_hidden_chart_oracle_at_8": 1.0130607387330142,
+    "ncar_to_proposal_oracle_at_8": 0.3633085481924293,
+    "outcome_ptr_at_8": 0.0,
+    "pairwise_causal_calibration_ece": 0.17529934581328913,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.42857142857142855,
+    "proposal_oracle_success_gain_over_base_at_8": -0.35714285714285715,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 0.9213141335972718,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.6676012235028403,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.17857142857142858,
+    "selected_success_gain_over_base_at_8": -0.6071428571428571,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.4545445234647819,
+    "selected_utility_gain_over_base_at_8": -1.1343708336353302,
+    "selector_gap_fraction_to_hidden_at_8": 0.004415993410754663,
+    "selector_regret_at_8": 0.4667696101324899,
+    "success_selector_gap_at_8": 0.25,
+    "success_support_gap_at_8": 0.5714285714285714,
+    "success_total_car_to_hidden_at_8": 0.8214285714285714,
+    "support_gap_at_8": 1.0786858664027281,
+    "support_gap_fraction_to_hidden_at_8": 1.0086447506040905,
+    "total_car_to_hidden_at_8": 1.5454554765352182
+  },
+  "StackCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.2222222222222222,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.7696565665580608,
+    "branch_car_at_8": 0.1789455551792074,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 0.8148148148148148,
+    "hidden_chart_oracle_utility_at_8": 1.773819453186459,
+    "ncar_to_hidden_chart_oracle_at_8": 0.941800572990603,
+    "ncar_to_proposal_oracle_at_8": 7.855930598180106,
+    "outcome_ptr_at_8": 0.4444444444444444,
+    "pairwise_causal_calibration_ece": 0.2044928219534789,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.14814814814814814,
+    "proposal_oracle_success_gain_over_base_at_8": -0.07407407407407407,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 0.717987585398886,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.051668981159174884,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.07407407407407407,
+    "selected_success_gain_over_base_at_8": -0.14814814814814814,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.5390420302196786,
+    "selected_utility_gain_over_base_at_8": -0.2306145363383823,
+    "selector_gap_fraction_to_hidden_at_8": 0.10129949927260672,
+    "selector_regret_at_8": 0.1789455551792074,
+    "success_selector_gap_at_8": 0.07407407407407407,
+    "success_support_gap_at_8": 0.7037037037037037,
+    "success_total_car_to_hidden_at_8": 0.7777777777777778,
+    "support_gap_at_8": 1.0981735327729472,
+    "support_gap_fraction_to_hidden_at_8": 0.8405010826182241,
+    "total_car_to_hidden_at_8": 1.2771190879521546
+  }
+}

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2/measured_metrics/report.md

@@ -0,0 +1,40 @@
+# Metric Evaluation (measured)
+
+K: `8`
+
+| Metric | N | Micro mean | 95% CI | Task macro | Seed macro |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| any_generated_unsafe_known_at_8 | 144 | 0.3125 | [0.2431, 0.3889] | 0.1250 | 0.1639 |
+| base_safety_label_known | 144 | 1.0000 | [1.0000, 1.0000] | 1.0000 | 1.0000 |
+| base_success | 144 | 0.2986 | [0.2292, 0.3681] | 0.3035 | 0.3432 |
+| base_unsafe_known | 144 | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 |
+| base_utility | 144 | 0.7487 | [0.6186, 0.8692] | 0.7886 | 0.8603 |
+| branch_car_at_8 | 144 | 0.1883 | [0.1097, 0.2613] | 0.2371 | 0.2485 |
+| generated_safety_label_coverage_at_8 | 144 | 1.0000 | [1.0000, 1.0000] | 1.0000 | 1.0000 |
+| generated_unsafe_rate_known_at_8 | 144 | 0.2474 | [0.1858, 0.3281] | 0.0990 | 0.1191 |
+| hidden_chart_oracle_success_at_8 | 144 | 0.6597 | [0.5833, 0.7361] | 0.8407 | 0.8447 |
+| hidden_chart_oracle_utility_at_8 | 144 | 1.4151 | [1.2723, 1.5417] | 1.7378 | 1.7477 |
+| ncar_to_hidden_chart_oracle_at_8 | 103 | 2.2488 | [1.1836, 4.1038] | 1.3955 | 2.2277 |
+| ncar_to_proposal_oracle_at_8 | 121 | 2.0613 | [0.6453, 4.4401] | 1.9458 | 2.7290 |
+| outcome_ptr_at_8 | 144 | 0.2639 | [0.2014, 0.3403] | 0.2300 | 0.2246 |
+| pairwise_causal_calibration_ece | 144 | 0.2073 | [0.1883, 0.2282] | 0.1879 | 0.1933 |
+| proposal_oracle_safety_label_known_at_8 | 144 | 1.0000 | [1.0000, 1.0000] | 1.0000 | 1.0000 |
+| proposal_oracle_success_at_8 | 144 | 0.2014 | [0.1389, 0.2639] | 0.2461 | 0.2446 |
+| proposal_oracle_success_gain_over_base_at_8 | 144 | -0.0972 | [-0.1667, -0.0347] | -0.0574 | -0.0986 |
+| proposal_oracle_unsafe_known_at_8 | 144 | 0.2431 | [0.1875, 0.3194] | 0.0972 | 0.1225 |
+| proposal_oracle_utility_at_8 | 144 | 0.5701 | [0.4651, 0.7015] | 0.6720 | 0.6804 |
+| proposal_oracle_utility_gain_over_base_at_8 | 144 | -0.1786 | [-0.3059, -0.0614] | -0.1165 | -0.1799 |
+| selected_safety_label_known_at_8 | 144 | 1.0000 | [1.0000, 1.0000] | 1.0000 | 1.0000 |
+| selected_success_at_8 | 144 | 0.1042 | [0.0625, 0.1528] | 0.1072 | 0.1111 |
+| selected_success_gain_over_base_at_8 | 144 | -0.1944 | [-0.2708, -0.1250] | -0.1963 | -0.2321 |
+| selected_unsafe_known_at_8 | 144 | 0.2569 | [0.1875, 0.3403] | 0.1028 | 0.1178 |
+| selected_utility_at_8 | 144 | 0.3818 | [0.2986, 0.4771] | 0.4349 | 0.4319 |
+| selected_utility_gain_over_base_at_8 | 144 | -0.3669 | [-0.4965, -0.2505] | -0.3537 | -0.4284 |
+| selector_gap_fraction_to_hidden_at_8 | 103 | 0.5487 | [0.1597, 1.2471] | 0.2432 | 0.6417 |
+| selector_regret_at_8 | 144 | 0.1883 | [0.1097, 0.2613] | 0.2371 | 0.2485 |
+| success_selector_gap_at_8 | 144 | 0.0972 | [0.0486, 0.1458] | 0.1389 | 0.1335 |
+| success_support_gap_at_8 | 144 | 0.4653 | [0.3819, 0.5347] | 0.6020 | 0.6109 |
+| success_total_car_to_hidden_at_8 | 144 | 0.5625 | [0.4792, 0.6458] | 0.7410 | 0.7444 |
+| support_gap_at_8 | 144 | 0.8548 | [0.7165, 0.9951] | 1.0750 | 1.0802 |
+| support_gap_fraction_to_hidden_at_8 | 103 | 1.7083 | [0.9842, 2.8881] | 1.1552 | 1.5877 |
+| total_car_to_hidden_at_8 | 144 | 1.0429 | [0.9018, 1.1718] | 1.3120 | 1.3287 |

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2/measured_metrics/table.tex

@@ -0,0 +1,41 @@
+% Auto-generated by scripts/eval_metrics.py
+\begin{tabular}{lrrrr}
+\toprule
+Metric & N & Micro mean & CI low & CI high \\
+\midrule
+any\_generated\_unsafe\_known\_at\_8 & 144 & 0.3125 & 0.2431 & 0.3889 \\
+base\_safety\_label\_known & 144 & 1.0000 & 1.0000 & 1.0000 \\
+base\_success & 144 & 0.2986 & 0.2292 & 0.3681 \\
+base\_unsafe\_known & 144 & 0.0000 & 0.0000 & 0.0000 \\
+base\_utility & 144 & 0.7487 & 0.6186 & 0.8692 \\
+branch\_car\_at\_8 & 144 & 0.1883 & 0.1097 & 0.2613 \\
+generated\_safety\_label\_coverage\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
+generated\_unsafe\_rate\_known\_at\_8 & 144 & 0.2474 & 0.1858 & 0.3281 \\
+hidden\_chart\_oracle\_success\_at\_8 & 144 & 0.6597 & 0.5833 & 0.7361 \\
+hidden\_chart\_oracle\_utility\_at\_8 & 144 & 1.4151 & 1.2723 & 1.5417 \\
+ncar\_to\_hidden\_chart\_oracle\_at\_8 & 103 & 2.2488 & 1.1836 & 4.1038 \\
+ncar\_to\_proposal\_oracle\_at\_8 & 121 & 2.0613 & 0.6453 & 4.4401 \\
+outcome\_ptr\_at\_8 & 144 & 0.2639 & 0.2014 & 0.3403 \\
+pairwise\_causal\_calibration\_ece & 144 & 0.2073 & 0.1883 & 0.2282 \\
+proposal\_oracle\_safety\_label\_known\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
+proposal\_oracle\_success\_at\_8 & 144 & 0.2014 & 0.1389 & 0.2639 \\
+proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 144 & -0.0972 & -0.1667 & -0.0347 \\
+proposal\_oracle\_unsafe\_known\_at\_8 & 144 & 0.2431 & 0.1875 & 0.3194 \\
+proposal\_oracle\_utility\_at\_8 & 144 & 0.5701 & 0.4651 & 0.7015 \\
+proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 144 & -0.1786 & -0.3059 & -0.0614 \\
+selected\_safety\_label\_known\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
+selected\_success\_at\_8 & 144 & 0.1042 & 0.0625 & 0.1528 \\
+selected\_success\_gain\_over\_base\_at\_8 & 144 & -0.1944 & -0.2708 & -0.1250 \\
+selected\_unsafe\_known\_at\_8 & 144 & 0.2569 & 0.1875 & 0.3403 \\
+selected\_utility\_at\_8 & 144 & 0.3818 & 0.2986 & 0.4771 \\
+selected\_utility\_gain\_over\_base\_at\_8 & 144 & -0.3669 & -0.4965 & -0.2505 \\
+selector\_gap\_fraction\_to\_hidden\_at\_8 & 103 & 0.5487 & 0.1597 & 1.2471 \\
+selector\_regret\_at\_8 & 144 & 0.1883 & 0.1097 & 0.2613 \\
+success\_selector\_gap\_at\_8 & 144 & 0.0972 & 0.0486 & 0.1458 \\
+success\_support\_gap\_at\_8 & 144 & 0.4653 & 0.3819 & 0.5347 \\
+success\_total\_car\_to\_hidden\_at\_8 & 144 & 0.5625 & 0.4792 & 0.6458 \\
+support\_gap\_at\_8 & 144 & 0.8548 & 0.7165 & 0.9951 \\
+support\_gap\_fraction\_to\_hidden\_at\_8 & 103 & 1.7083 & 0.9842 & 2.8881 \\
+total\_car\_to\_hidden\_at\_8 & 144 & 1.0429 & 0.9018 & 1.1718 \\
+\bottomrule
+\end{tabular}

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2/metrics_by_task.json

@@ -0,0 +1,182 @@
+{
+  "LiftPegUpright-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.2,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.7833942204713822,
+    "branch_car_at_8": 0.25610943883657455,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 1.0,
+    "hidden_chart_oracle_utility_at_8": 2.0,
+    "ncar_to_hidden_chart_oracle_at_8": 0.8673945494371688,
+    "ncar_to_proposal_oracle_at_8": 0.22809248414000916,
+    "outcome_ptr_at_8": 0.4,
+    "pairwise_causal_calibration_ece": 0.18048328241453365,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.4,
+    "proposal_oracle_success_gain_over_base_at_8": 0.2,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 1.0454350337386131,
+    "proposal_oracle_utility_gain_over_base_at_8": 0.262040813267231,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.2,
+    "selected_success_gain_over_base_at_8": 0.0,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.7893255949020386,
+    "selected_utility_gain_over_base_at_8": 0.005931374430656433,
+    "selector_gap_fraction_to_hidden_at_8": 0.2500967471724982,
+    "selector_regret_at_8": 0.25610943883657455,
+    "success_selector_gap_at_8": 0.2,
+    "success_support_gap_at_8": 0.6,
+    "success_total_car_to_hidden_at_8": 0.8,
+    "support_gap_at_8": 0.9545649662613869,
+    "support_gap_fraction_to_hidden_at_8": 0.617297807934756,
+    "total_car_to_hidden_at_8": 1.2106744050979614
+  },
+  "PickCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.14285714285714285,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.353121066732066,
+    "branch_car_at_8": 0.21211611798831395,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 1.0,
+    "hidden_chart_oracle_utility_at_8": 2.0,
+    "ncar_to_hidden_chart_oracle_at_8": 1.0109060531044503,
+    "ncar_to_proposal_oracle_at_8": 0.52622527077833,
+    "outcome_ptr_at_8": 0.0,
+    "pairwise_causal_calibration_ece": 0.14872269547853015,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.14285714285714285,
+    "proposal_oracle_success_gain_over_base_at_8": 0.0,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 0.341122750725065,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.01199831600700106,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.0,
+    "selected_success_gain_over_base_at_8": -0.14285714285714285,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.129006632736751,
+    "selected_utility_gain_over_base_at_8": -0.224114433995315,
+    "selector_gap_fraction_to_hidden_at_8": 0.003587880851496335,
+    "selector_regret_at_8": 0.21211611798831395,
+    "success_selector_gap_at_8": 0.14285714285714285,
+    "success_support_gap_at_8": 0.8571428571428571,
+    "success_total_car_to_hidden_at_8": 1.0,
+    "support_gap_at_8": 1.658877249274935,
+    "support_gap_fraction_to_hidden_at_8": 1.0073181775152928,
+    "total_car_to_hidden_at_8": 1.870993367263249
+  },
+  "PullCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.625,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.16666666666666666,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.4476979454047978,
+    "branch_car_at_8": 0.07172732797334902,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.4947916666666667,
+    "hidden_chart_oracle_success_at_8": 0.3888888888888889,
+    "hidden_chart_oracle_utility_at_8": 0.9149795592659049,
+    "ncar_to_hidden_chart_oracle_at_8": 3.144587613695753,
+    "ncar_to_proposal_oracle_at_8": 0.755293067551312,
+    "outcome_ptr_at_8": 0.3055555555555556,
+    "pairwise_causal_calibration_ece": 0.23026700740431424,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.1111111111111111,
+    "proposal_oracle_success_gain_over_base_at_8": -0.05555555555555555,
+    "proposal_oracle_unsafe_known_at_8": 0.4861111111111111,
+    "proposal_oracle_utility_at_8": 0.3342518941693318,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.11344605123546596,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.08333333333333333,
+    "selected_success_gain_over_base_at_8": -0.08333333333333333,
+    "selected_unsafe_known_at_8": 0.5138888888888888,
+    "selected_utility_at_8": 0.2625245661959828,
+    "selected_utility_gain_over_base_at_8": -0.18517337920881496,
+    "selector_gap_fraction_to_hidden_at_8": 0.8563952330010932,
+    "selector_regret_at_8": 0.07172732797334902,
+    "success_selector_gap_at_8": 0.027777777777777776,
+    "success_support_gap_at_8": 0.2777777777777778,
+    "success_total_car_to_hidden_at_8": 0.3055555555555556,
+    "support_gap_at_8": 0.5844958343142126,
+    "support_gap_fraction_to_hidden_at_8": 2.3021517980465727,
+    "total_car_to_hidden_at_8": 0.6557380551918565
+  },
+  "PushCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.7857142857142857,
+    "base_unsafe_known": 0.0,
+    "base_utility": 1.588915357100112,
+    "branch_car_at_8": 0.4667696101324899,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 1.0,
+    "hidden_chart_oracle_utility_at_8": 2.0,
+    "ncar_to_hidden_chart_oracle_at_8": 1.0130607387330142,
+    "ncar_to_proposal_oracle_at_8": 0.3633085481924293,
+    "outcome_ptr_at_8": 0.0,
+    "pairwise_causal_calibration_ece": 0.17529934581328913,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.42857142857142855,
+    "proposal_oracle_success_gain_over_base_at_8": -0.35714285714285715,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 0.9213141335972718,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.6676012235028403,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.17857142857142858,
+    "selected_success_gain_over_base_at_8": -0.6071428571428571,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.4545445234647819,
+    "selected_utility_gain_over_base_at_8": -1.1343708336353302,
+    "selector_gap_fraction_to_hidden_at_8": 0.004415993410754663,
+    "selector_regret_at_8": 0.4667696101324899,
+    "success_selector_gap_at_8": 0.25,
+    "success_support_gap_at_8": 0.5714285714285714,
+    "success_total_car_to_hidden_at_8": 0.8214285714285714,
+    "support_gap_at_8": 1.0786858664027281,
+    "support_gap_fraction_to_hidden_at_8": 1.0086447506040905,
+    "total_car_to_hidden_at_8": 1.5454554765352182
+  },
+  "StackCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.2222222222222222,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.7696565665580608,
+    "branch_car_at_8": 0.1789455551792074,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 0.8148148148148148,
+    "hidden_chart_oracle_utility_at_8": 1.773819453186459,
+    "ncar_to_hidden_chart_oracle_at_8": 0.941800572990603,
+    "ncar_to_proposal_oracle_at_8": 7.855930598180106,
+    "outcome_ptr_at_8": 0.4444444444444444,
+    "pairwise_causal_calibration_ece": 0.2044928219534789,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.14814814814814814,
+    "proposal_oracle_success_gain_over_base_at_8": -0.07407407407407407,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 0.717987585398886,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.051668981159174884,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.07407407407407407,
+    "selected_success_gain_over_base_at_8": -0.14814814814814814,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.5390420302196786,
+    "selected_utility_gain_over_base_at_8": -0.2306145363383823,
+    "selector_gap_fraction_to_hidden_at_8": 0.10129949927260672,
+    "selector_regret_at_8": 0.1789455551792074,
+    "success_selector_gap_at_8": 0.07407407407407407,
+    "success_support_gap_at_8": 0.7037037037037037,
+    "success_total_car_to_hidden_at_8": 0.7777777777777778,
+    "support_gap_at_8": 1.0981735327729472,
+    "support_gap_fraction_to_hidden_at_8": 0.8405010826182241,
+    "total_car_to_hidden_at_8": 1.2771190879521546
+  }
+}

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2/report.md

@@ -0,0 +1,13 @@
+# CTT Generated Measured Rollout
+
+Rows: `144`
+K: `8`
+Checkpoint: `runs/ctt_residual_base_context_obs_seed2/model.pt`
+
+| Metric | Mean |
+| --- | ---: |
+| OutcomePTR@K | 0.2639 |
+| SelectorRegret@K | 0.1883 |
+| SupportGap@K | 0.8450 |
+
+These are measured simulator rollouts of decoded CTT candidates, not PPTC proxies.

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2/source_data_hash.txt

@@ -0,0 +1 @@
+67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2/source_split_hash.txt

@@ -0,0 +1 @@
+867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2/split_hash.txt

@@ -0,0 +1 @@
+867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2/table.tex

@@ -0,0 +1,41 @@
+% Auto-generated by scripts/eval_metrics.py
+\begin{tabular}{lrrrr}
+\toprule
+Metric & N & Micro mean & CI low & CI high \\
+\midrule
+any\_generated\_unsafe\_known\_at\_8 & 144 & 0.3125 & 0.2431 & 0.3889 \\
+base\_safety\_label\_known & 144 & 1.0000 & 1.0000 & 1.0000 \\
+base\_success & 144 & 0.2986 & 0.2292 & 0.3681 \\
+base\_unsafe\_known & 144 & 0.0000 & 0.0000 & 0.0000 \\
+base\_utility & 144 & 0.7487 & 0.6186 & 0.8692 \\
+branch\_car\_at\_8 & 144 & 0.1883 & 0.1097 & 0.2613 \\
+generated\_safety\_label\_coverage\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
+generated\_unsafe\_rate\_known\_at\_8 & 144 & 0.2474 & 0.1858 & 0.3281 \\
+hidden\_chart\_oracle\_success\_at\_8 & 144 & 0.6597 & 0.5833 & 0.7361 \\
+hidden\_chart\_oracle\_utility\_at\_8 & 144 & 1.4151 & 1.2723 & 1.5417 \\
+ncar\_to\_hidden\_chart\_oracle\_at\_8 & 103 & 2.2488 & 1.1836 & 4.1038 \\
+ncar\_to\_proposal\_oracle\_at\_8 & 121 & 2.0613 & 0.6453 & 4.4401 \\
+outcome\_ptr\_at\_8 & 144 & 0.2639 & 0.2014 & 0.3403 \\
+pairwise\_causal\_calibration\_ece & 144 & 0.2073 & 0.1883 & 0.2282 \\
+proposal\_oracle\_safety\_label\_known\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
+proposal\_oracle\_success\_at\_8 & 144 & 0.2014 & 0.1389 & 0.2639 \\
+proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 144 & -0.0972 & -0.1667 & -0.0347 \\
+proposal\_oracle\_unsafe\_known\_at\_8 & 144 & 0.2431 & 0.1875 & 0.3194 \\
+proposal\_oracle\_utility\_at\_8 & 144 & 0.5701 & 0.4651 & 0.7015 \\
+proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 144 & -0.1786 & -0.3059 & -0.0614 \\
+selected\_safety\_label\_known\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
+selected\_success\_at\_8 & 144 & 0.1042 & 0.0625 & 0.1528 \\
+selected\_success\_gain\_over\_base\_at\_8 & 144 & -0.1944 & -0.2708 & -0.1250 \\
+selected\_unsafe\_known\_at\_8 & 144 & 0.2569 & 0.1875 & 0.3403 \\
+selected\_utility\_at\_8 & 144 & 0.3818 & 0.2986 & 0.4771 \\
+selected\_utility\_gain\_over\_base\_at\_8 & 144 & -0.3669 & -0.4965 & -0.2505 \\
+selector\_gap\_fraction\_to\_hidden\_at\_8 & 103 & 0.5487 & 0.1597 & 1.2471 \\
+selector\_regret\_at\_8 & 144 & 0.1883 & 0.1097 & 0.2613 \\
+success\_selector\_gap\_at\_8 & 144 & 0.0972 & 0.0486 & 0.1458 \\
+success\_support\_gap\_at\_8 & 144 & 0.4653 & 0.3819 & 0.5347 \\
+success\_total\_car\_to\_hidden\_at\_8 & 144 & 0.5625 & 0.4792 & 0.6458 \\
+support\_gap\_at\_8 & 144 & 0.8548 & 0.7165 & 0.9951 \\
+support\_gap\_fraction\_to\_hidden\_at\_8 & 103 & 1.7083 & 0.9842 & 2.8881 \\
+total\_car\_to\_hidden\_at\_8 & 144 & 1.0429 & 0.9018 & 1.1718 \\
+\bottomrule
+\end{tabular}

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed2/train.log

@@ -0,0 +1 @@
+CTT checkpoint trained separately; rollout used checkpoint=runs/ctt_residual_base_context_obs_seed2/model.pt