Upload per-dim trainmax train seed1 artifact

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1/command.txt

@@ -0,0 +1 @@
+python scripts/eval_ctt_generated_rollout.py --checkpoint runs/ctt_residual_base_context_obs_seed1/model.pt --source-index data/cil_charts_rgb_refs/train/index.json --target-index data/cil_charts_rgb_refs/train/index.json --out-dir runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1 --k 8 --pool-size 0 --neighbors 8 --max-target-charts 144 --group-batch-size 1 --device cpu --sim-backend physx_cpu --render-backend cpu --restore-tolerance 1e-5 --delta-scale 1.0 --execution-action-scale 1.0 --execution-action-scale-vector 0.381401261606,0.371117178171,0.354308917074,0.237850081835,0.248271283428,0.232648812617,0.215420039658 --execution-action-transform identity --bootstrap-samples 200 --exclude-self-source --disable-action-clipping

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1/config.yaml

@@ -0,0 +1,22 @@
+bootstrap_samples: 200
+checkpoint: runs/ctt_residual_base_context_obs_seed1/model.pt
+delta_scale: 1.0
+device: cpu
+disable_action_clipping: True
+exclude_self_source: True
+execution_action_scale: 1.0
+execution_action_scale_vector: 0.381401261606,0.371117178171,0.354308917074,0.237850081835,0.248271283428,0.232648812617,0.215420039658
+execution_action_transform: identity
+group_batch_size: 1
+include_targets_without_positives: False
+k: 8
+max_target_charts: 144
+neighbors: 8
+out_dir: runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1
+pool_size: 0
+render_backend: cpu
+restore_tolerance: 1e-05
+sim_backend: physx_cpu
+skip_metrics: False
+source_index: data/cil_charts_rgb_refs/train/index.json
+target_index: data/cil_charts_rgb_refs/train/index.json

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1/data_hash.txt

@@ -0,0 +1 @@
+67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1/git_hash.txt

@@ -0,0 +1 @@
+

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1/measured_metrics/metrics_by_task.json

@@ -0,0 +1,182 @@
+{
+  "LiftPegUpright-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.2,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.7833942204713822,
+    "branch_car_at_8": 0.14423791021108628,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 1.0,
+    "hidden_chart_oracle_utility_at_8": 2.0,
+    "ncar_to_hidden_chart_oracle_at_8": 1.0046663903117472,
+    "ncar_to_proposal_oracle_at_8": 1.1821539208817973,
+    "outcome_ptr_at_8": 0.5,
+    "pairwise_causal_calibration_ece": 0.20637651536789478,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.1,
+    "proposal_oracle_success_gain_over_base_at_8": -0.1,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 0.6632065117359162,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.120187708735466,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.0,
+    "selected_success_gain_over_base_at_8": -0.2,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.5189686015248298,
+    "selected_utility_gain_over_base_at_8": -0.26442561894655225,
+    "selector_gap_fraction_to_hidden_at_8": 0.0131493360597599,
+    "selector_regret_at_8": 0.14423791021108628,
+    "success_selector_gap_at_8": 0.1,
+    "success_support_gap_at_8": 0.9,
+    "success_total_car_to_hidden_at_8": 1.0,
+    "support_gap_at_8": 1.336793488264084,
+    "support_gap_fraction_to_hidden_at_8": 0.9915170609894026,
+    "total_car_to_hidden_at_8": 1.4810313984751702
+  },
+  "PickCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.14285714285714285,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.353121066732066,
+    "branch_car_at_8": 0.006391581147909164,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 1.0,
+    "hidden_chart_oracle_utility_at_8": 2.0,
+    "ncar_to_hidden_chart_oracle_at_8": 1.0108391911468655,
+    "ncar_to_proposal_oracle_at_8": 0.38623213869813006,
+    "outcome_ptr_at_8": 0.0,
+    "pairwise_causal_calibration_ece": 0.12307543867823803,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.0,
+    "proposal_oracle_success_gain_over_base_at_8": -0.14285714285714285,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 0.13091773965529033,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.22220332707677568,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.0,
+    "selected_success_gain_over_base_at_8": -0.14285714285714285,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.12452615850738116,
+    "selected_utility_gain_over_base_at_8": -0.22859490822468484,
+    "selector_gap_fraction_to_hidden_at_8": 0.0024959160053537595,
+    "selector_regret_at_8": 0.006391581147909164,
+    "success_selector_gap_at_8": 0.0,
+    "success_support_gap_at_8": 1.0,
+    "success_total_car_to_hidden_at_8": 1.0,
+    "support_gap_at_8": 1.8690822603447097,
+    "support_gap_fraction_to_hidden_at_8": 1.0083432804033443,
+    "total_car_to_hidden_at_8": 1.8754738414926189
+  },
+  "PullCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.5277777777777778,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.16666666666666666,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.4476979454047978,
+    "branch_car_at_8": 0.0978457247838378,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.4479166666666667,
+    "hidden_chart_oracle_success_at_8": 0.3888888888888889,
+    "hidden_chart_oracle_utility_at_8": 0.9149795592659049,
+    "ncar_to_hidden_chart_oracle_at_8": 3.4444329592209466,
+    "ncar_to_proposal_oracle_at_8": 1.3695755329843828,
+    "outcome_ptr_at_8": 0.25,
+    "pairwise_causal_calibration_ece": 0.21769475103072738,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.1111111111111111,
+    "proposal_oracle_success_gain_over_base_at_8": -0.05555555555555555,
+    "proposal_oracle_unsafe_known_at_8": 0.4305555555555556,
+    "proposal_oracle_utility_at_8": 0.33873097791284734,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.10896696749195042,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.06944444444444445,
+    "selected_success_gain_over_base_at_8": -0.09722222222222222,
+    "selected_unsafe_known_at_8": 0.4027777777777778,
+    "selected_utility_at_8": 0.24088525312900957,
+    "selected_utility_gain_over_base_at_8": -0.20681269227578822,
+    "selector_gap_fraction_to_hidden_at_8": 0.5813030142387842,
+    "selector_regret_at_8": 0.0978457247838378,
+    "success_selector_gap_at_8": 0.041666666666666664,
+    "success_support_gap_at_8": 0.2916666666666667,
+    "success_total_car_to_hidden_at_8": 0.3194444444444444,
+    "support_gap_at_8": 0.6059644525844811,
+    "support_gap_fraction_to_hidden_at_8": 2.9200906228028263,
+    "total_car_to_hidden_at_8": 0.6777591987176694
+  },
+  "PushCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.7857142857142857,
+    "base_unsafe_known": 0.0,
+    "base_utility": 1.588915357100112,
+    "branch_car_at_8": 0.009395991052900041,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 1.0,
+    "hidden_chart_oracle_utility_at_8": 2.0,
+    "ncar_to_hidden_chart_oracle_at_8": 1.0079970339438908,
+    "ncar_to_proposal_oracle_at_8": 0.4704038201575328,
+    "outcome_ptr_at_8": 0.10714285714285714,
+    "pairwise_causal_calibration_ece": 0.17648360505045407,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.25,
+    "proposal_oracle_success_gain_over_base_at_8": -0.5357142857142857,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 0.5998135374060699,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.9891018196940422,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.25,
+    "selected_success_gain_over_base_at_8": -0.5357142857142857,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.5904175463531699,
+    "selected_utility_gain_over_base_at_8": -0.9984978107469422,
+    "selector_gap_fraction_to_hidden_at_8": 0.008142780936455294,
+    "selector_regret_at_8": 0.009395991052900041,
+    "success_selector_gap_at_8": 0.0,
+    "success_support_gap_at_8": 0.75,
+    "success_total_car_to_hidden_at_8": 0.75,
+    "support_gap_at_8": 1.4001864625939302,
+    "support_gap_fraction_to_hidden_at_8": 0.9998542582629938,
+    "total_car_to_hidden_at_8": 1.4095824536468302
+  },
+  "StackCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.2222222222222222,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.7696565665580608,
+    "branch_car_at_8": 0.4238730266138359,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 0.8148148148148148,
+    "hidden_chart_oracle_utility_at_8": 1.773819453186459,
+    "ncar_to_hidden_chart_oracle_at_8": 0.9821689575201963,
+    "ncar_to_proposal_oracle_at_8": 0.8999523449762592,
+    "outcome_ptr_at_8": 0.4444444444444444,
+    "pairwise_causal_calibration_ece": 0.21327293794799237,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.37037037037037035,
+    "proposal_oracle_success_gain_over_base_at_8": 0.14814814814814814,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 0.9812490476502312,
+    "proposal_oracle_utility_gain_over_base_at_8": 0.21159248109217044,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.1111111111111111,
+    "selected_success_gain_over_base_at_8": -0.1111111111111111,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.5573760210363953,
+    "selected_utility_gain_over_base_at_8": -0.2122805455216655,
+    "selector_gap_fraction_to_hidden_at_8": 0.26653641497673003,
+    "selector_regret_at_8": 0.4238730266138359,
+    "success_selector_gap_at_8": 0.25925925925925924,
+    "success_support_gap_at_8": 0.5185185185185185,
+    "success_total_car_to_hidden_at_8": 0.7407407407407407,
+    "support_gap_at_8": 0.8799223436249627,
+    "support_gap_fraction_to_hidden_at_8": 0.7610870972609242,
+    "total_car_to_hidden_at_8": 1.258785097135438
+  }
+}

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1/measured_metrics/report.md

@@ -0,0 +1,40 @@
+# Metric Evaluation (measured)
+
+K: `8`
+
+| Metric | N | Micro mean | 95% CI | Task macro | Seed macro |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| any_generated_unsafe_known_at_8 | 144 | 0.2639 | [0.1944, 0.3542] | 0.1056 | 0.1259 |
+| base_safety_label_known | 144 | 1.0000 | [1.0000, 1.0000] | 1.0000 | 1.0000 |
+| base_success | 144 | 0.2986 | [0.2292, 0.3681] | 0.3035 | 0.3432 |
+| base_unsafe_known | 144 | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 |
+| base_utility | 144 | 0.7487 | [0.6186, 0.8692] | 0.7886 | 0.8603 |
+| branch_car_at_8 | 144 | 0.1406 | [0.0769, 0.2112] | 0.1363 | 0.1493 |
+| generated_safety_label_coverage_at_8 | 144 | 1.0000 | [1.0000, 1.0000] | 1.0000 | 1.0000 |
+| generated_unsafe_rate_known_at_8 | 144 | 0.2240 | [0.1615, 0.3099] | 0.0896 | 0.1064 |
+| hidden_chart_oracle_success_at_8 | 144 | 0.6597 | [0.5833, 0.7361] | 0.8407 | 0.8447 |
+| hidden_chart_oracle_utility_at_8 | 144 | 1.4151 | [1.2723, 1.5417] | 1.7378 | 1.7477 |
+| ncar_to_hidden_chart_oracle_at_8 | 103 | 2.4453 | [1.3081, 4.0284] | 1.4900 | 1.9044 |
+| ncar_to_proposal_oracle_at_8 | 123 | 1.0735 | [0.6790, 1.7494] | 0.8617 | 0.8129 |
+| outcome_ptr_at_8 | 144 | 0.2639 | [0.1944, 0.3264] | 0.2603 | 0.2621 |
+| pairwise_causal_calibration_ece | 144 | 0.2035 | [0.1796, 0.2269] | 0.1874 | 0.1945 |
+| proposal_oracle_safety_label_known_at_8 | 144 | 1.0000 | [1.0000, 1.0000] | 1.0000 | 1.0000 |
+| proposal_oracle_success_at_8 | 144 | 0.1806 | [0.1111, 0.2500] | 0.1663 | 0.2069 |
+| proposal_oracle_success_gain_over_base_at_8 | 144 | -0.1181 | [-0.1806, -0.0556] | -0.1372 | -0.1363 |
+| proposal_oracle_unsafe_known_at_8 | 144 | 0.2153 | [0.1458, 0.2986] | 0.0861 | 0.0935 |
+| proposal_oracle_utility_at_8 | 144 | 0.5224 | [0.4014, 0.6465] | 0.5428 | 0.5986 |
+| proposal_oracle_utility_gain_over_base_at_8 | 144 | -0.2263 | [-0.3332, -0.1178] | -0.2458 | -0.2617 |
+| selected_safety_label_known_at_8 | 144 | 1.0000 | [1.0000, 1.0000] | 1.0000 | 1.0000 |
+| selected_success_at_8 | 144 | 0.1042 | [0.0625, 0.1597] | 0.0861 | 0.1200 |
+| selected_success_gain_over_base_at_8 | 144 | -0.1944 | [-0.2778, -0.1181] | -0.2174 | -0.2232 |
+| selected_unsafe_known_at_8 | 144 | 0.2014 | [0.1319, 0.2847] | 0.0806 | 0.0903 |
+| selected_utility_at_8 | 144 | 0.3818 | [0.3002, 0.4882] | 0.4064 | 0.4493 |
+| selected_utility_gain_over_base_at_8 | 144 | -0.3668 | [-0.5087, -0.2414] | -0.3821 | -0.4110 |
+| selector_gap_fraction_to_hidden_at_8 | 103 | 0.4028 | [0.2256, 0.6623] | 0.1743 | 0.2473 |
+| selector_regret_at_8 | 144 | 0.1406 | [0.0769, 0.2112] | 0.1363 | 0.1493 |
+| success_selector_gap_at_8 | 144 | 0.0764 | [0.0347, 0.1181] | 0.0802 | 0.0869 |
+| success_support_gap_at_8 | 144 | 0.5000 | [0.4236, 0.5764] | 0.6920 | 0.6566 |
+| success_total_car_to_hidden_at_8 | 144 | 0.5625 | [0.4792, 0.6458] | 0.7620 | 0.7355 |
+| support_gap_at_8 | 144 | 0.9239 | [0.7863, 1.0600] | 1.2184 | 1.1735 |
+| support_gap_fraction_to_hidden_at_8 | 103 | 2.0859 | [1.0987, 3.6285] | 1.3362 | 1.6768 |
+| total_car_to_hidden_at_8 | 144 | 1.0430 | [0.8962, 1.1790] | 1.3405 | 1.3113 |

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1/measured_metrics/table.tex

@@ -0,0 +1,41 @@
+% Auto-generated by scripts/eval_metrics.py
+\begin{tabular}{lrrrr}
+\toprule
+Metric & N & Micro mean & CI low & CI high \\
+\midrule
+any\_generated\_unsafe\_known\_at\_8 & 144 & 0.2639 & 0.1944 & 0.3542 \\
+base\_safety\_label\_known & 144 & 1.0000 & 1.0000 & 1.0000 \\
+base\_success & 144 & 0.2986 & 0.2292 & 0.3681 \\
+base\_unsafe\_known & 144 & 0.0000 & 0.0000 & 0.0000 \\
+base\_utility & 144 & 0.7487 & 0.6186 & 0.8692 \\
+branch\_car\_at\_8 & 144 & 0.1406 & 0.0769 & 0.2112 \\
+generated\_safety\_label\_coverage\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
+generated\_unsafe\_rate\_known\_at\_8 & 144 & 0.2240 & 0.1615 & 0.3099 \\
+hidden\_chart\_oracle\_success\_at\_8 & 144 & 0.6597 & 0.5833 & 0.7361 \\
+hidden\_chart\_oracle\_utility\_at\_8 & 144 & 1.4151 & 1.2723 & 1.5417 \\
+ncar\_to\_hidden\_chart\_oracle\_at\_8 & 103 & 2.4453 & 1.3081 & 4.0284 \\
+ncar\_to\_proposal\_oracle\_at\_8 & 123 & 1.0735 & 0.6790 & 1.7494 \\
+outcome\_ptr\_at\_8 & 144 & 0.2639 & 0.1944 & 0.3264 \\
+pairwise\_causal\_calibration\_ece & 144 & 0.2035 & 0.1796 & 0.2269 \\
+proposal\_oracle\_safety\_label\_known\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
+proposal\_oracle\_success\_at\_8 & 144 & 0.1806 & 0.1111 & 0.2500 \\
+proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 144 & -0.1181 & -0.1806 & -0.0556 \\
+proposal\_oracle\_unsafe\_known\_at\_8 & 144 & 0.2153 & 0.1458 & 0.2986 \\
+proposal\_oracle\_utility\_at\_8 & 144 & 0.5224 & 0.4014 & 0.6465 \\
+proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 144 & -0.2263 & -0.3332 & -0.1178 \\
+selected\_safety\_label\_known\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
+selected\_success\_at\_8 & 144 & 0.1042 & 0.0625 & 0.1597 \\
+selected\_success\_gain\_over\_base\_at\_8 & 144 & -0.1944 & -0.2778 & -0.1181 \\
+selected\_unsafe\_known\_at\_8 & 144 & 0.2014 & 0.1319 & 0.2847 \\
+selected\_utility\_at\_8 & 144 & 0.3818 & 0.3002 & 0.4882 \\
+selected\_utility\_gain\_over\_base\_at\_8 & 144 & -0.3668 & -0.5087 & -0.2414 \\
+selector\_gap\_fraction\_to\_hidden\_at\_8 & 103 & 0.4028 & 0.2256 & 0.6623 \\
+selector\_regret\_at\_8 & 144 & 0.1406 & 0.0769 & 0.2112 \\
+success\_selector\_gap\_at\_8 & 144 & 0.0764 & 0.0347 & 0.1181 \\
+success\_support\_gap\_at\_8 & 144 & 0.5000 & 0.4236 & 0.5764 \\
+success\_total\_car\_to\_hidden\_at\_8 & 144 & 0.5625 & 0.4792 & 0.6458 \\
+support\_gap\_at\_8 & 144 & 0.9239 & 0.7863 & 1.0600 \\
+support\_gap\_fraction\_to\_hidden\_at\_8 & 103 & 2.0859 & 1.0987 & 3.6285 \\
+total\_car\_to\_hidden\_at\_8 & 144 & 1.0430 & 0.8962 & 1.1790 \\
+\bottomrule
+\end{tabular}

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1/metrics_by_task.json

@@ -0,0 +1,182 @@
+{
+  "LiftPegUpright-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.2,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.7833942204713822,
+    "branch_car_at_8": 0.14423791021108628,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 1.0,
+    "hidden_chart_oracle_utility_at_8": 2.0,
+    "ncar_to_hidden_chart_oracle_at_8": 1.0046663903117472,
+    "ncar_to_proposal_oracle_at_8": 1.1821539208817973,
+    "outcome_ptr_at_8": 0.5,
+    "pairwise_causal_calibration_ece": 0.20637651536789478,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.1,
+    "proposal_oracle_success_gain_over_base_at_8": -0.1,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 0.6632065117359162,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.120187708735466,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.0,
+    "selected_success_gain_over_base_at_8": -0.2,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.5189686015248298,
+    "selected_utility_gain_over_base_at_8": -0.26442561894655225,
+    "selector_gap_fraction_to_hidden_at_8": 0.0131493360597599,
+    "selector_regret_at_8": 0.14423791021108628,
+    "success_selector_gap_at_8": 0.1,
+    "success_support_gap_at_8": 0.9,
+    "success_total_car_to_hidden_at_8": 1.0,
+    "support_gap_at_8": 1.336793488264084,
+    "support_gap_fraction_to_hidden_at_8": 0.9915170609894026,
+    "total_car_to_hidden_at_8": 1.4810313984751702
+  },
+  "PickCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.14285714285714285,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.353121066732066,
+    "branch_car_at_8": 0.006391581147909164,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 1.0,
+    "hidden_chart_oracle_utility_at_8": 2.0,
+    "ncar_to_hidden_chart_oracle_at_8": 1.0108391911468655,
+    "ncar_to_proposal_oracle_at_8": 0.38623213869813006,
+    "outcome_ptr_at_8": 0.0,
+    "pairwise_causal_calibration_ece": 0.12307543867823803,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.0,
+    "proposal_oracle_success_gain_over_base_at_8": -0.14285714285714285,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 0.13091773965529033,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.22220332707677568,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.0,
+    "selected_success_gain_over_base_at_8": -0.14285714285714285,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.12452615850738116,
+    "selected_utility_gain_over_base_at_8": -0.22859490822468484,
+    "selector_gap_fraction_to_hidden_at_8": 0.0024959160053537595,
+    "selector_regret_at_8": 0.006391581147909164,
+    "success_selector_gap_at_8": 0.0,
+    "success_support_gap_at_8": 1.0,
+    "success_total_car_to_hidden_at_8": 1.0,
+    "support_gap_at_8": 1.8690822603447097,
+    "support_gap_fraction_to_hidden_at_8": 1.0083432804033443,
+    "total_car_to_hidden_at_8": 1.8754738414926189
+  },
+  "PullCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.5277777777777778,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.16666666666666666,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.4476979454047978,
+    "branch_car_at_8": 0.0978457247838378,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.4479166666666667,
+    "hidden_chart_oracle_success_at_8": 0.3888888888888889,
+    "hidden_chart_oracle_utility_at_8": 0.9149795592659049,
+    "ncar_to_hidden_chart_oracle_at_8": 3.4444329592209466,
+    "ncar_to_proposal_oracle_at_8": 1.3695755329843828,
+    "outcome_ptr_at_8": 0.25,
+    "pairwise_causal_calibration_ece": 0.21769475103072738,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.1111111111111111,
+    "proposal_oracle_success_gain_over_base_at_8": -0.05555555555555555,
+    "proposal_oracle_unsafe_known_at_8": 0.4305555555555556,
+    "proposal_oracle_utility_at_8": 0.33873097791284734,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.10896696749195042,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.06944444444444445,
+    "selected_success_gain_over_base_at_8": -0.09722222222222222,
+    "selected_unsafe_known_at_8": 0.4027777777777778,
+    "selected_utility_at_8": 0.24088525312900957,
+    "selected_utility_gain_over_base_at_8": -0.20681269227578822,
+    "selector_gap_fraction_to_hidden_at_8": 0.5813030142387842,
+    "selector_regret_at_8": 0.0978457247838378,
+    "success_selector_gap_at_8": 0.041666666666666664,
+    "success_support_gap_at_8": 0.2916666666666667,
+    "success_total_car_to_hidden_at_8": 0.3194444444444444,
+    "support_gap_at_8": 0.6059644525844811,
+    "support_gap_fraction_to_hidden_at_8": 2.9200906228028263,
+    "total_car_to_hidden_at_8": 0.6777591987176694
+  },
+  "PushCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.7857142857142857,
+    "base_unsafe_known": 0.0,
+    "base_utility": 1.588915357100112,
+    "branch_car_at_8": 0.009395991052900041,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 1.0,
+    "hidden_chart_oracle_utility_at_8": 2.0,
+    "ncar_to_hidden_chart_oracle_at_8": 1.0079970339438908,
+    "ncar_to_proposal_oracle_at_8": 0.4704038201575328,
+    "outcome_ptr_at_8": 0.10714285714285714,
+    "pairwise_causal_calibration_ece": 0.17648360505045407,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.25,
+    "proposal_oracle_success_gain_over_base_at_8": -0.5357142857142857,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 0.5998135374060699,
+    "proposal_oracle_utility_gain_over_base_at_8": -0.9891018196940422,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.25,
+    "selected_success_gain_over_base_at_8": -0.5357142857142857,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.5904175463531699,
+    "selected_utility_gain_over_base_at_8": -0.9984978107469422,
+    "selector_gap_fraction_to_hidden_at_8": 0.008142780936455294,
+    "selector_regret_at_8": 0.009395991052900041,
+    "success_selector_gap_at_8": 0.0,
+    "success_support_gap_at_8": 0.75,
+    "success_total_car_to_hidden_at_8": 0.75,
+    "support_gap_at_8": 1.4001864625939302,
+    "support_gap_fraction_to_hidden_at_8": 0.9998542582629938,
+    "total_car_to_hidden_at_8": 1.4095824536468302
+  },
+  "StackCube-v1": {
+    "any_generated_unsafe_known_at_8": 0.0,
+    "base_safety_label_known": 1.0,
+    "base_success": 0.2222222222222222,
+    "base_unsafe_known": 0.0,
+    "base_utility": 0.7696565665580608,
+    "branch_car_at_8": 0.4238730266138359,
+    "generated_safety_label_coverage_at_8": 1.0,
+    "generated_unsafe_rate_known_at_8": 0.0,
+    "hidden_chart_oracle_success_at_8": 0.8148148148148148,
+    "hidden_chart_oracle_utility_at_8": 1.773819453186459,
+    "ncar_to_hidden_chart_oracle_at_8": 0.9821689575201963,
+    "ncar_to_proposal_oracle_at_8": 0.8999523449762592,
+    "outcome_ptr_at_8": 0.4444444444444444,
+    "pairwise_causal_calibration_ece": 0.21327293794799237,
+    "proposal_oracle_safety_label_known_at_8": 1.0,
+    "proposal_oracle_success_at_8": 0.37037037037037035,
+    "proposal_oracle_success_gain_over_base_at_8": 0.14814814814814814,
+    "proposal_oracle_unsafe_known_at_8": 0.0,
+    "proposal_oracle_utility_at_8": 0.9812490476502312,
+    "proposal_oracle_utility_gain_over_base_at_8": 0.21159248109217044,
+    "selected_safety_label_known_at_8": 1.0,
+    "selected_success_at_8": 0.1111111111111111,
+    "selected_success_gain_over_base_at_8": -0.1111111111111111,
+    "selected_unsafe_known_at_8": 0.0,
+    "selected_utility_at_8": 0.5573760210363953,
+    "selected_utility_gain_over_base_at_8": -0.2122805455216655,
+    "selector_gap_fraction_to_hidden_at_8": 0.26653641497673003,
+    "selector_regret_at_8": 0.4238730266138359,
+    "success_selector_gap_at_8": 0.25925925925925924,
+    "success_support_gap_at_8": 0.5185185185185185,
+    "success_total_car_to_hidden_at_8": 0.7407407407407407,
+    "support_gap_at_8": 0.8799223436249627,
+    "support_gap_fraction_to_hidden_at_8": 0.7610870972609242,
+    "total_car_to_hidden_at_8": 1.258785097135438
+  }
+}

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1/report.md

@@ -0,0 +1,13 @@
+# CTT Generated Measured Rollout
+
+Rows: `144`
+K: `8`
+Checkpoint: `runs/ctt_residual_base_context_obs_seed1/model.pt`
+
+| Metric | Mean |
+| --- | ---: |
+| OutcomePTR@K | 0.2639 |
+| SelectorRegret@K | 0.1406 |
+| SupportGap@K | 0.8927 |
+
+These are measured simulator rollouts of decoded CTT candidates, not PPTC proxies.

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1/source_data_hash.txt

@@ -0,0 +1 @@
+67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1/source_split_hash.txt

@@ -0,0 +1 @@
+867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1/split_hash.txt

@@ -0,0 +1 @@
+867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1/table.tex

@@ -0,0 +1,41 @@
+% Auto-generated by scripts/eval_metrics.py
+\begin{tabular}{lrrrr}
+\toprule
+Metric & N & Micro mean & CI low & CI high \\
+\midrule
+any\_generated\_unsafe\_known\_at\_8 & 144 & 0.2639 & 0.1944 & 0.3542 \\
+base\_safety\_label\_known & 144 & 1.0000 & 1.0000 & 1.0000 \\
+base\_success & 144 & 0.2986 & 0.2292 & 0.3681 \\
+base\_unsafe\_known & 144 & 0.0000 & 0.0000 & 0.0000 \\
+base\_utility & 144 & 0.7487 & 0.6186 & 0.8692 \\
+branch\_car\_at\_8 & 144 & 0.1406 & 0.0769 & 0.2112 \\
+generated\_safety\_label\_coverage\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
+generated\_unsafe\_rate\_known\_at\_8 & 144 & 0.2240 & 0.1615 & 0.3099 \\
+hidden\_chart\_oracle\_success\_at\_8 & 144 & 0.6597 & 0.5833 & 0.7361 \\
+hidden\_chart\_oracle\_utility\_at\_8 & 144 & 1.4151 & 1.2723 & 1.5417 \\
+ncar\_to\_hidden\_chart\_oracle\_at\_8 & 103 & 2.4453 & 1.3081 & 4.0284 \\
+ncar\_to\_proposal\_oracle\_at\_8 & 123 & 1.0735 & 0.6790 & 1.7494 \\
+outcome\_ptr\_at\_8 & 144 & 0.2639 & 0.1944 & 0.3264 \\
+pairwise\_causal\_calibration\_ece & 144 & 0.2035 & 0.1796 & 0.2269 \\
+proposal\_oracle\_safety\_label\_known\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
+proposal\_oracle\_success\_at\_8 & 144 & 0.1806 & 0.1111 & 0.2500 \\
+proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 144 & -0.1181 & -0.1806 & -0.0556 \\
+proposal\_oracle\_unsafe\_known\_at\_8 & 144 & 0.2153 & 0.1458 & 0.2986 \\
+proposal\_oracle\_utility\_at\_8 & 144 & 0.5224 & 0.4014 & 0.6465 \\
+proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 144 & -0.2263 & -0.3332 & -0.1178 \\
+selected\_safety\_label\_known\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
+selected\_success\_at\_8 & 144 & 0.1042 & 0.0625 & 0.1597 \\
+selected\_success\_gain\_over\_base\_at\_8 & 144 & -0.1944 & -0.2778 & -0.1181 \\
+selected\_unsafe\_known\_at\_8 & 144 & 0.2014 & 0.1319 & 0.2847 \\
+selected\_utility\_at\_8 & 144 & 0.3818 & 0.3002 & 0.4882 \\
+selected\_utility\_gain\_over\_base\_at\_8 & 144 & -0.3668 & -0.5087 & -0.2414 \\
+selector\_gap\_fraction\_to\_hidden\_at\_8 & 103 & 0.4028 & 0.2256 & 0.6623 \\
+selector\_regret\_at\_8 & 144 & 0.1406 & 0.0769 & 0.2112 \\
+success\_selector\_gap\_at\_8 & 144 & 0.0764 & 0.0347 & 0.1181 \\
+success\_support\_gap\_at\_8 & 144 & 0.5000 & 0.4236 & 0.5764 \\
+success\_total\_car\_to\_hidden\_at\_8 & 144 & 0.5625 & 0.4792 & 0.6458 \\
+support\_gap\_at\_8 & 144 & 0.9239 & 0.7863 & 1.0600 \\
+support\_gap\_fraction\_to\_hidden\_at\_8 & 103 & 2.0859 & 1.0987 & 3.6285 \\
+total\_car\_to\_hidden\_at\_8 & 144 & 1.0430 & 0.8962 & 1.1790 \\
+\bottomrule
+\end{tabular}

workspace/runs/ctt_residual_base_context_obs_rollout_train_cal_perdim_trainmax_seed1/train.log

@@ -0,0 +1 @@
+CTT checkpoint trained separately; rollout used checkpoint=runs/ctt_residual_base_context_obs_seed1/model.pt