auto-sync 2026-07-04T04:28:21Z workspace (part 3)

workspace/runs/ctt_base_context_obs_dominance_tanh_train_to_test/command.txt

@@ -1 +1 @@
-python scripts/eval_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_tanh_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_tanh_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --score-source row --out-dir runs/ctt_base_context_obs_dominance_tanh_train_to_test --k 8 --bootstrap-samples 1000
+python scripts/eval_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_tanh_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_tanh_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --score-source row --out-dir runs/ctt_base_context_obs_dominance_tanh_train_to_test --k 8 --bootstrap-samples 1000 --no-markdown-report

workspace/runs/ctt_base_context_obs_dominance_tanh_train_to_test/config.yaml

@@ -6,6 +6,7 @@ checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt
 eval_input: runs/ctt_base_context_obs_test_tanh_rollout_comparison/combined_measured_candidates.json
 eval_target_index: data/cil_charts_rgb_refs/test/index.json
 k: 8
+no_markdown_report: True
 out_dir: runs/ctt_base_context_obs_dominance_tanh_train_to_test
 score_source: row
 tau: auto

workspace/runs/ctt_base_context_obs_dominance_tanh_train_to_test/metrics_by_task.json

@@ -1,9 +1,13 @@
 {
   "LiftPegUpright-v1": {
     "base_predicted_score": 3.8269843657811484,
+    "base_safety_label_known": 1.0,
     "base_success": 0.6,
+    "base_unsafe_known": 0.0,
     "base_utility": 1.4457374572753907,
     "branch_car": 0.0165030837059021,
+    "candidate_safety_label_coverage": 1.0,
+    "candidate_unsafe_rate_known": 0.0,
     "coverage": 0.0,
     "execute_generated": 0.0,
     "fallback_rate": 1.0,
@@ -13,11 +17,17 @@
     "lcb_margin": -4.858325332899889,
     "measured_margin": -0.896817363301913,
     "outcome_ptr": 0.13333333333333333,
+    "pairwise_causal_calibration_accuracy": 0.5174362674362675,
+    "pairwise_causal_calibration_confidence": 0.6127088559366983,
+    "pairwise_causal_calibration_ece": 0.18290187299277877,
+    "pairwise_causal_calibration_pairs": 26.266666666666666,
     "predicted_margin": 1.045054308573405,
     "proposal_oracle_success": 0.4666666666666667,
     "proposal_oracle_utility": 1.1608628551165263,
+    "selected_safety_label_known": 1.0,
     "selected_success": 0.6,
     "selected_success_gain_over_base": 0.0,
+    "selected_unsafe_known": 0.0,
     "selected_utility": 1.4457374572753907,
     "selected_utility_gain_over_base": 0.0,
     "selector_regret": 0.0165030837059021,
@@ -25,16 +35,24 @@
     "success_support_gap": 0.5333333333333333,
     "success_total_car_to_hidden": 0.4,
     "support_gap": 0.8391371448834737,
+    "top_candidate_safety_label_known": 1.0,
+    "top_candidate_unsafe_known": 0.0,
     "top_generated_success": 0.06666666666666667,
     "top_generated_utility": 0.5489200939734776,
     "top_index": 0.0,
-    "top_predicted_score": 4.872038674354553
+    "top_predicted_score": 4.872038674354553,
+    "unsafe_execution_known": 0.0,
+    "unsafe_execution_label_known": 1.0
   },
   "PickCube-v1": {
     "base_predicted_score": 5.362143039703369,
+    "base_safety_label_known": 1.0,
     "base_success": 0.0,
+    "base_unsafe_known": 0.0,
     "base_utility": 0.584423840045929,
     "branch_car": 0.0,
+    "candidate_safety_label_coverage": 1.0,
+    "candidate_unsafe_rate_known": 0.0,
     "coverage": 0.0,
     "execute_generated": 0.0,
     "fallback_rate": 1.0,
@@ -44,11 +62,17 @@
     "lcb_margin": -5.801814439396064,
     "measured_margin": -0.47414494802554447,
     "outcome_ptr": 0.0,
+    "pairwise_causal_calibration_accuracy": 0.4523809523809524,
+    "pairwise_causal_calibration_confidence": 0.6407613179383564,
+    "pairwise_causal_calibration_ece": 0.2697729681159227,
+    "pairwise_causal_calibration_pairs": 28.0,
     "predicted_margin": 0.10156520207722981,
     "proposal_oracle_success": 0.0,
     "proposal_oracle_utility": 0.1308078020811081,
+    "selected_safety_label_known": 1.0,
     "selected_success": 0.0,
     "selected_success_gain_over_base": 0.0,
+    "selected_unsafe_known": 0.0,
     "selected_utility": 0.584423840045929,
     "selected_utility_gain_over_base": 0.0,
     "selector_regret": 0.0,
@@ -56,16 +80,24 @@
     "success_support_gap": 1.0,
     "success_total_car_to_hidden": 1.0,
     "support_gap": 1.869192197918892,
+    "top_candidate_safety_label_known": 1.0,
+    "top_candidate_unsafe_known": 0.0,
     "top_generated_success": 0.0,
     "top_generated_utility": 0.11027889202038448,
     "top_index": 0.0,
-    "top_predicted_score": 5.463708241780599
+    "top_predicted_score": 5.463708241780599,
+    "unsafe_execution_known": 0.0,
+    "unsafe_execution_label_known": 1.0
   },
   "PullCube-v1": {
     "base_predicted_score": 2.447210181752841,
+    "base_safety_label_known": 1.0,
     "base_success": 0.4,
+    "base_unsafe_known": 0.0,
     "base_utility": 0.8622376645449549,
     "branch_car": 0.28954911112474896,
+    "candidate_safety_label_coverage": 1.0,
+    "candidate_unsafe_rate_known": 0.0,
     "coverage": 0.17333333333333334,
     "execute_generated": 0.17333333333333334,
     "fallback_rate": 0.8266666666666667,
@@ -75,11 +107,17 @@
     "lcb_margin": -3.2941473240653676,
     "measured_margin": -0.35265780616454623,
     "outcome_ptr": 0.38666666666666666,
+    "pairwise_causal_calibration_accuracy": 0.542498076826435,
+    "pairwise_causal_calibration_confidence": 0.6704810777825188,
+    "pairwise_causal_calibration_ece": 0.20693605964037032,
+    "pairwise_causal_calibration_pairs": 24.133333333333333,
     "predicted_margin": 2.6092323174079257,
     "proposal_oracle_success": 0.3333333333333333,
     "proposal_oracle_utility": 0.7346437490296861,
+    "selected_safety_label_known": 1.0,
     "selected_success": 0.37333333333333335,
     "selected_success_gain_over_base": -0.02666666666666667,
+    "selected_unsafe_known": 0.0,
     "selected_utility": 0.8076825904628883,
     "selected_utility_gain_over_base": -0.05455507408206661,
     "selector_regret": 0.28954911112474896,
@@ -87,16 +125,24 @@
     "success_support_gap": 0.2,
     "success_total_car_to_hidden": 0.2,
     "support_gap": 0.43614903089279927,
+    "top_candidate_safety_label_known": 1.0,
+    "top_candidate_unsafe_known": 0.0,
     "top_generated_success": 0.22666666666666666,
     "top_generated_utility": 0.5095798583804086,
     "top_index": 0.0,
-    "top_predicted_score": 5.056442499160767
+    "top_predicted_score": 5.056442499160767,
+    "unsafe_execution_known": 0.0,
+    "unsafe_execution_label_known": 1.0
   },
   "PushCube-v1": {
     "base_predicted_score": 2.019581354326672,
+    "base_safety_label_known": 1.0,
     "base_success": 0.16666666666666666,
+    "base_unsafe_known": 0.0,
     "base_utility": 0.3684890990455945,
     "branch_car": 0.8920534153779348,
+    "candidate_safety_label_coverage": 1.0,
+    "candidate_unsafe_rate_known": 0.0,
     "coverage": 0.2222222222222222,
     "execute_generated": 0.2222222222222222,
     "fallback_rate": 0.7777777777777778,
@@ -106,11 +152,17 @@
     "lcb_margin": -1.9659957745008998,
     "measured_margin": 0.7824923925929599,
     "outcome_ptr": 0.8333333333333334,
+    "pairwise_causal_calibration_accuracy": 0.5254250841750842,
+    "pairwise_causal_calibration_confidence": 0.6640246392315027,
+    "pairwise_causal_calibration_ece": 0.2003164242374059,
+    "pairwise_causal_calibration_pairs": 19.055555555555557,
     "predicted_margin": 3.9373838669723935,
     "proposal_oracle_success": 0.6111111111111112,
     "proposal_oracle_utility": 1.2712341505620215,
+    "selected_safety_label_known": 1.0,
     "selected_success": 0.16666666666666666,
     "selected_success_gain_over_base": 0.0,
+    "selected_unsafe_known": 0.0,
     "selected_utility": 0.3791807351840867,
     "selected_utility_gain_over_base": 0.01069163613849216,
     "selector_regret": 0.8920534153779348,
@@ -118,16 +170,24 @@
     "success_support_gap": 0.3333333333333333,
     "success_total_car_to_hidden": 0.6666666666666666,
     "support_gap": 0.6292925361129973,
+    "top_candidate_safety_label_known": 1.0,
+    "top_candidate_unsafe_known": 0.0,
     "top_generated_success": 0.5555555555555556,
     "top_generated_utility": 1.1509814916385546,
     "top_index": 0.0,
-    "top_predicted_score": 5.956965221299066
+    "top_predicted_score": 5.956965221299066,
+    "unsafe_execution_known": 0.0,
+    "unsafe_execution_label_known": 1.0
   },
   "StackCube-v1": {
     "base_predicted_score": 5.208852457277702,
+    "base_safety_label_known": 1.0,
     "base_success": 0.18181818181818182,
+    "base_unsafe_known": 0.0,
     "base_utility": 0.6242787648331035,
     "branch_car": 0.25828172898653784,
+    "candidate_safety_label_coverage": 1.0,
+    "candidate_unsafe_rate_known": 0.0,
     "coverage": 0.0,
     "execute_generated": 0.0,
     "fallback_rate": 1.0,
@@ -137,11 +197,17 @@
     "lcb_margin": -5.010546159337867,
     "measured_margin": -0.04017875275828622,
     "outcome_ptr": 0.42424242424242425,
+    "pairwise_causal_calibration_accuracy": 0.5363842218387673,
+    "pairwise_causal_calibration_confidence": 0.632474250754894,
+    "pairwise_causal_calibration_ece": 0.21190325771990542,
+    "pairwise_causal_calibration_pairs": 24.818181818181817,
     "predicted_margin": 0.8928334821354259,
     "proposal_oracle_success": 0.36363636363636365,
     "proposal_oracle_utility": 0.8693077586817018,
+    "selected_safety_label_known": 1.0,
     "selected_success": 0.18181818181818182,
     "selected_success_gain_over_base": 0.0,
+    "selected_unsafe_known": 0.0,
     "selected_utility": 0.6242787648331035,
     "selected_utility_gain_over_base": 0.0,
     "selector_regret": 0.25828172898653784,
@@ -149,9 +215,13 @@
     "success_support_gap": 0.6363636363636364,
     "success_total_car_to_hidden": 0.8181818181818182,
     "support_gap": 1.130692241318298,
+    "top_candidate_safety_label_known": 1.0,
+    "top_candidate_unsafe_known": 0.0,
     "top_generated_success": 0.24242424242424243,
     "top_generated_utility": 0.5841000120748173,
     "top_index": 0.0,
-    "top_predicted_score": 6.101685939413128
+    "top_predicted_score": 6.101685939413128,
+    "unsafe_execution_known": 0.0,
+    "unsafe_execution_label_known": 1.0
   }
 }

workspace/runs/ctt_base_context_obs_dominance_tanh_train_to_test/table.tex

@@ -1,8 +1,8 @@
 % Auto-generated by scripts/eval_dominance_selector.py
-\begin{tabular}{lrrrrrrrr}
+\begin{tabular}{lrrrrrrrrrr}
 \toprule
-Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap \\
+Rows & Coverage & Fallback & Unsafe exec. & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap & Cal. ECE \\
 \midrule
-144 & 0.1181 & 0.8819 & 0.3333 & 0.3194 & 0.3819 & 0.4167 & 0.3681 & 0.1736 \\
+144 & 0.1181 & 0.8819 & 0.0000 & 0.3333 & 0.3194 & 0.3819 & 0.4167 & 0.3681 & 0.1736 & 0.1224 \\
 \bottomrule
 \end{tabular}