auto-sync 2026-07-04T07:51:42Z workspace (part 3)

workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/eval.log

@@ -0,0 +1,3 @@
+evaluated learned fallback rule on held-out measured rollout rows
+eval_input=runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json
+num_eval_rows=144

workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/metrics_by_task.json

@@ -0,0 +1,137 @@
+{
+  "LiftPegUpright-v1": {
+    "base_success": 0.2,
+    "base_utility": 0.9498705387115478,
+    "coverage": 0.6,
+    "execute_generated": 0.6,
+    "fallback_rate": 0.4,
+    "hidden_chart_oracle_success": 1.0,
+    "hidden_chart_oracle_utility": 2.0,
+    "outcome_ptr": 0.5333333333333333,
+    "pairwise_causal_calibration_accuracy": 0.54512467498401,
+    "pairwise_causal_calibration_confidence": 0.54564531858913,
+    "pairwise_causal_calibration_ece": 0.11030342503862721,
+    "pairwise_causal_calibration_pairs": 94.8,
+    "predicted_margin": 0.14233301795724493,
+    "proposal_oracle_success": 0.6,
+    "proposal_oracle_utility": 1.4122399787108104,
+    "selected_candidate_index": 7.6,
+    "selected_success": 0.13333333333333333,
+    "selected_success_gain_over_base": -0.06666666666666667,
+    "selected_utility": 0.8185293744007747,
+    "selected_utility_gain_over_base": -0.13134116431077322,
+    "selector_regret": 0.6451034953196844,
+    "success_selector_gap": 0.4666666666666667,
+    "success_support_gap": 0.4,
+    "support_gap": 0.5877600212891897,
+    "tau": 0.0940318405174132
+  },
+  "PickCube-v1": {
+    "base_success": 0.0,
+    "base_utility": 0.5839591026306152,
+    "coverage": 0.0,
+    "execute_generated": 0.0,
+    "fallback_rate": 1.0,
+    "hidden_chart_oracle_success": 1.0,
+    "hidden_chart_oracle_utility": 2.0,
+    "outcome_ptr": 0.0,
+    "pairwise_causal_calibration_accuracy": 0.39814814814814814,
+    "pairwise_causal_calibration_confidence": 0.5607943587261041,
+    "pairwise_causal_calibration_ece": 0.16599791089703783,
+    "pairwise_causal_calibration_pairs": 36.0,
+    "predicted_margin": -0.39039029063840003,
+    "proposal_oracle_success": 0.0,
+    "proposal_oracle_utility": 0.13425676027933756,
+    "selected_candidate_index": 5.0,
+    "selected_success": 0.0,
+    "selected_success_gain_over_base": 0.0,
+    "selected_utility": 0.5839591026306152,
+    "selected_utility_gain_over_base": 0.0,
+    "selector_regret": 0.0,
+    "success_selector_gap": 0.0,
+    "success_support_gap": 1.0,
+    "support_gap": 1.8657432397206624,
+    "tau": 0.3611323919348086
+  },
+  "PullCube-v1": {
+    "base_success": 0.36,
+    "base_utility": 0.7751704612234608,
+    "coverage": 0.6266666666666667,
+    "execute_generated": 0.6266666666666667,
+    "fallback_rate": 0.37333333333333335,
+    "hidden_chart_oracle_success": 0.52,
+    "hidden_chart_oracle_utility": 1.1467878860235214,
+    "outcome_ptr": 0.4666666666666667,
+    "pairwise_causal_calibration_accuracy": 0.4995022986920425,
+    "pairwise_causal_calibration_confidence": 0.5422481003539483,
+    "pairwise_causal_calibration_ece": 0.11105368029396932,
+    "pairwise_causal_calibration_pairs": 92.32,
+    "predicted_margin": 0.18136174528296545,
+    "proposal_oracle_success": 0.5333333333333333,
+    "proposal_oracle_utility": 1.1093241602430741,
+    "selected_candidate_index": 7.1866666666666665,
+    "selected_success": 0.32,
+    "selected_success_gain_over_base": -0.04,
+    "selected_utility": 0.6814040022250265,
+    "selected_utility_gain_over_base": -0.09376645899843425,
+    "selector_regret": 0.5039629975054413,
+    "success_selector_gap": 0.25333333333333335,
+    "success_support_gap": 0.02666666666666667,
+    "support_gap": 0.1075624164690574,
+    "tau": 0.10332537601561143
+  },
+  "PushCube-v1": {
+    "base_success": 0.16666666666666666,
+    "base_utility": 0.3924041787783305,
+    "coverage": 0.6666666666666666,
+    "execute_generated": 0.6666666666666666,
+    "fallback_rate": 0.3333333333333333,
+    "hidden_chart_oracle_success": 0.8333333333333334,
+    "hidden_chart_oracle_utility": 1.7015800600250561,
+    "outcome_ptr": 0.8333333333333334,
+    "pairwise_causal_calibration_accuracy": 0.588120956369862,
+    "pairwise_causal_calibration_confidence": 0.5338991876592735,
+    "pairwise_causal_calibration_ece": 0.08423604919725729,
+    "pairwise_causal_calibration_pairs": 97.44444444444444,
+    "predicted_margin": 0.4582899698155127,
+    "proposal_oracle_success": 0.8888888888888888,
+    "proposal_oracle_utility": 1.7857814149724112,
+    "selected_candidate_index": 7.055555555555555,
+    "selected_success": 0.5,
+    "selected_success_gain_over_base": 0.3333333333333333,
+    "selected_utility": 1.062640056014061,
+    "selected_utility_gain_over_base": 0.6702358772357305,
+    "selector_regret": 0.7231413589583503,
+    "success_selector_gap": 0.3888888888888889,
+    "success_support_gap": 0.1111111111111111,
+    "support_gap": 0.21421858502758873,
+    "tau": 0.35241155802994506
+  },
+  "StackCube-v1": {
+    "base_success": 0.2727272727272727,
+    "base_utility": 0.7763947045261209,
+    "coverage": 0.6060606060606061,
+    "execute_generated": 0.6060606060606061,
+    "fallback_rate": 0.3939393939393939,
+    "hidden_chart_oracle_success": 1.0,
+    "hidden_chart_oracle_utility": 2.0,
+    "outcome_ptr": 0.6363636363636364,
+    "pairwise_causal_calibration_accuracy": 0.5309413874218266,
+    "pairwise_causal_calibration_confidence": 0.5440067619410754,
+    "pairwise_causal_calibration_ece": 0.09869174841720094,
+    "pairwise_causal_calibration_pairs": 67.45454545454545,
+    "predicted_margin": 0.06420154895093595,
+    "proposal_oracle_success": 0.5151515151515151,
+    "proposal_oracle_utility": 1.1567951707225856,
+    "selected_candidate_index": 6.212121212121212,
+    "selected_success": 0.30303030303030304,
+    "selected_success_gain_over_base": 0.030303030303030304,
+    "selected_utility": 0.8023195727304979,
+    "selected_utility_gain_over_base": 0.025924868204376915,
+    "selector_regret": 0.3561627133326097,
+    "success_selector_gap": 0.21212121212121213,
+    "success_support_gap": 0.48484848484848486,
+    "support_gap": 0.8432048292774142,
+    "tau": 0.0009112489076126273
+  }
+}

workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/table.tex

@@ -0,0 +1,8 @@
+% Auto-generated by scripts/eval_learned_dominance_selector.py
+\begin{tabular}{lrrrrrrrrr}
+\toprule
+Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap & Cal. ECE \\
+\midrule
+144 & 0.6111 & 0.3889 & 0.2917 & 0.3125 & 0.5694 & 0.5486 & 0.2014 & 0.2778 & 0.0255 \\
+\bottomrule
+\end{tabular}

workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/train.log

@@ -0,0 +1,5 @@
+trained ridge dominance calibrator on calibration measured rows only
+fit_objective=pointwise
+selected_lambda=10.0
+tau={"LiftPegUpright-v1": 0.094032, "PickCube-v1": 0.361132, "PullCube-v1": 0.103325, "PushCube-v1": 0.352412, "StackCube-v1": 0.000911, "__global__": 0.113201}
+calibration_input=runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json

workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/eval.log

@@ -0,0 +1,3 @@
+evaluated learned fallback rule on held-out measured rollout rows
+eval_input=runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json
+num_eval_rows=144

workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/metrics_by_task.json

@@ -0,0 +1,137 @@
+{
+  "LiftPegUpright-v1": {
+    "base_success": 0.2,
+    "base_utility": 0.9498705387115478,
+    "coverage": 0.6,
+    "execute_generated": 0.6,
+    "fallback_rate": 0.4,
+    "hidden_chart_oracle_success": 1.0,
+    "hidden_chart_oracle_utility": 2.0,
+    "outcome_ptr": 0.5333333333333333,
+    "pairwise_causal_calibration_accuracy": 0.5555136707566375,
+    "pairwise_causal_calibration_confidence": 0.5720692082177231,
+    "pairwise_causal_calibration_ece": 0.11666816996110206,
+    "pairwise_causal_calibration_pairs": 94.8,
+    "predicted_margin": 0.7829724526600295,
+    "proposal_oracle_success": 0.6,
+    "proposal_oracle_utility": 1.4122399787108104,
+    "selected_candidate_index": 5.133333333333334,
+    "selected_success": 0.06666666666666667,
+    "selected_success_gain_over_base": -0.13333333333333333,
+    "selected_utility": 0.7418742964665095,
+    "selected_utility_gain_over_base": -0.20799624224503835,
+    "selector_regret": 0.7217585732539494,
+    "success_selector_gap": 0.5333333333333333,
+    "success_support_gap": 0.4,
+    "support_gap": 0.5877600212891897,
+    "tau": 0.7478358458742457
+  },
+  "PickCube-v1": {
+    "base_success": 0.0,
+    "base_utility": 0.5839591026306152,
+    "coverage": 0.0,
+    "execute_generated": 0.0,
+    "fallback_rate": 1.0,
+    "hidden_chart_oracle_success": 1.0,
+    "hidden_chart_oracle_utility": 2.0,
+    "outcome_ptr": 0.0,
+    "pairwise_causal_calibration_accuracy": 0.3333333333333333,
+    "pairwise_causal_calibration_confidence": 0.6023940721845762,
+    "pairwise_causal_calibration_ece": 0.2745681321943875,
+    "pairwise_causal_calibration_pairs": 36.0,
+    "predicted_margin": -0.12247428613675211,
+    "proposal_oracle_success": 0.0,
+    "proposal_oracle_utility": 0.13425676027933756,
+    "selected_candidate_index": 2.0,
+    "selected_success": 0.0,
+    "selected_success_gain_over_base": 0.0,
+    "selected_utility": 0.5839591026306152,
+    "selected_utility_gain_over_base": 0.0,
+    "selector_regret": 0.0,
+    "success_selector_gap": 0.0,
+    "success_support_gap": 1.0,
+    "support_gap": 1.8657432397206624,
+    "tau": 1.280136139242778
+  },
+  "PullCube-v1": {
+    "base_success": 0.36,
+    "base_utility": 0.7751704612234608,
+    "coverage": 0.52,
+    "execute_generated": 0.52,
+    "fallback_rate": 0.48,
+    "hidden_chart_oracle_success": 0.52,
+    "hidden_chart_oracle_utility": 1.1467878860235214,
+    "outcome_ptr": 0.4666666666666667,
+    "pairwise_causal_calibration_accuracy": 0.5062744920994355,
+    "pairwise_causal_calibration_confidence": 0.5796546075128097,
+    "pairwise_causal_calibration_ece": 0.12491436687229007,
+    "pairwise_causal_calibration_pairs": 92.32,
+    "predicted_margin": 0.9547622553200451,
+    "proposal_oracle_success": 0.5333333333333333,
+    "proposal_oracle_utility": 1.1093241602430741,
+    "selected_candidate_index": 7.266666666666667,
+    "selected_success": 0.32,
+    "selected_success_gain_over_base": -0.04,
+    "selected_utility": 0.6846797956805676,
+    "selected_utility_gain_over_base": -0.09049066554289312,
+    "selector_regret": 0.45504458223469557,
+    "success_selector_gap": 0.22666666666666666,
+    "success_support_gap": 0.02666666666666667,
+    "support_gap": 0.1075624164690574,
+    "tau": 0.9470671281950384
+  },
+  "PushCube-v1": {
+    "base_success": 0.16666666666666666,
+    "base_utility": 0.3924041787783305,
+    "coverage": 0.8888888888888888,
+    "execute_generated": 0.8888888888888888,
+    "fallback_rate": 0.1111111111111111,
+    "hidden_chart_oracle_success": 0.8333333333333334,
+    "hidden_chart_oracle_utility": 1.7015800600250561,
+    "outcome_ptr": 0.8333333333333334,
+    "pairwise_causal_calibration_accuracy": 0.5428451728329846,
+    "pairwise_causal_calibration_confidence": 0.6000598858100837,
+    "pairwise_causal_calibration_ece": 0.15543541117172033,
+    "pairwise_causal_calibration_pairs": 97.44444444444444,
+    "predicted_margin": 1.843134202495152,
+    "proposal_oracle_success": 0.8888888888888888,
+    "proposal_oracle_utility": 1.7857814149724112,
+    "selected_candidate_index": 6.222222222222222,
+    "selected_success": 0.3888888888888889,
+    "selected_success_gain_over_base": 0.2222222222222222,
+    "selected_utility": 0.8422558406988779,
+    "selected_utility_gain_over_base": 0.4498516619205475,
+    "selector_regret": 0.9435255742735333,
+    "success_selector_gap": 0.5,
+    "success_support_gap": 0.1111111111111111,
+    "support_gap": 0.21421858502758873,
+    "tau": 1.2264360294338525
+  },
+  "StackCube-v1": {
+    "base_success": 0.2727272727272727,
+    "base_utility": 0.7763947045261209,
+    "coverage": 0.5151515151515151,
+    "execute_generated": 0.5151515151515151,
+    "fallback_rate": 0.48484848484848486,
+    "hidden_chart_oracle_success": 1.0,
+    "hidden_chart_oracle_utility": 2.0,
+    "outcome_ptr": 0.6363636363636364,
+    "pairwise_causal_calibration_accuracy": 0.5286305231046764,
+    "pairwise_causal_calibration_confidence": 0.5839574678118631,
+    "pairwise_causal_calibration_ece": 0.13914695546983782,
+    "pairwise_causal_calibration_pairs": 67.45454545454545,
+    "predicted_margin": 0.832831201132664,
+    "proposal_oracle_success": 0.5151515151515151,
+    "proposal_oracle_utility": 1.1567951707225856,
+    "selected_candidate_index": 6.606060606060606,
+    "selected_success": 0.30303030303030304,
+    "selected_success_gain_over_base": 0.030303030303030304,
+    "selected_utility": 0.7861830118027601,
+    "selected_utility_gain_over_base": 0.00978830727663907,
+    "selector_regret": 0.37342702303871966,
+    "success_selector_gap": 0.21212121212121213,
+    "success_support_gap": 0.48484848484848486,
+    "support_gap": 0.8432048292774142,
+    "tau": 0.8938079363311323
+  }
+}

workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/table.tex

@@ -0,0 +1,8 @@
+% Auto-generated by scripts/eval_learned_dominance_selector.py
+\begin{tabular}{lrrrrrrrrr}
+\toprule
+Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap & Cal. ECE \\
+\midrule
+144 & 0.5625 & 0.4375 & 0.2917 & 0.2917 & 0.5694 & 0.5486 & 0.2014 & 0.2847 & 0.0611 \\
+\bottomrule
+\end{tabular}

workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/train.log

@@ -0,0 +1,5 @@
+trained ridge dominance calibrator on calibration measured rows only
+fit_objective=pointwise
+selected_lambda=1.0
+tau={"LiftPegUpright-v1": 0.747836, "PickCube-v1": 1.280136, "PullCube-v1": 0.947067, "PushCube-v1": 1.226436, "StackCube-v1": 0.893808, "__global__": 0.906341}
+calibration_input=runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json