manual-selector-job-sync 2026-07-04T04:36:59Z workspace (part 5)

workspace/runs/ctt_base_context_obs_learned_dominance_basic_perdim_trainmax_train_to_test/command.txt

@@ -0,0 +1 @@
+python scripts/eval_learned_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_learned_dominance_basic_perdim_trainmax_train_to_test --feature-set basic --target utility_margin --k 8 --bootstrap-samples 1000 --no-markdown-report

workspace/runs/ctt_base_context_obs_learned_dominance_basic_perdim_trainmax_train_to_test/config.yaml

@@ -0,0 +1,18 @@
+bootstrap_samples: 1000
+calibration_input: runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json
+calibration_target_index: data/cil_charts_rgb_refs/train/index.json
+checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt
+eval_input: runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json
+eval_target_index: data/cil_charts_rgb_refs/test/index.json
+feature_set: basic
+fit_objective: pointwise
+k: 8
+no_markdown_report: True
+out_dir: runs/ctt_base_context_obs_learned_dominance_basic_perdim_trainmax_train_to_test
+pairwise_weight: 1.0
+ridge_lambdas: 0,0.01,0.1,1,10,100
+selector_chart_feature_mode: base_context_obs_obj
+source_index: None
+success_bonus: 1.0
+target: utility_margin
+threshold_scope: global

workspace/runs/ctt_base_context_obs_learned_dominance_basic_perdim_trainmax_train_to_test/data_hash.txt

@@ -0,0 +1,6 @@
+{
+  "calibration_input": "130d86b8c5ba5dc901ee48835454693bf42c361803c01f6b2d80590cb2493491",
+  "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85",
+  "eval_input": "88d4bfe8c95f8d1579cbe59096cf665dc5f2dfdd99481a386a25e1f3cfa2b09d",
+  "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36"
+}

workspace/runs/ctt_base_context_obs_learned_dominance_basic_perdim_trainmax_train_to_test/eval.log

@@ -0,0 +1,3 @@
+evaluated learned fallback rule on held-out measured rollout rows
+eval_input=runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json
+num_eval_rows=144

workspace/runs/ctt_base_context_obs_learned_dominance_basic_perdim_trainmax_train_to_test/git_hash.txt

@@ -0,0 +1 @@
+14e81292e2004925f81e0bb4810f6f144bfd87b7

workspace/runs/ctt_base_context_obs_learned_dominance_basic_perdim_trainmax_train_to_test/metrics_by_task.json

@@ -0,0 +1,137 @@
+{
+  "LiftPegUpright-v1": {
+    "base_success": 0.0,
+    "base_utility": 0.4796089559793472,
+    "coverage": 0.4,
+    "execute_generated": 0.4,
+    "fallback_rate": 0.6,
+    "hidden_chart_oracle_success": 1.0,
+    "hidden_chart_oracle_utility": 2.0,
+    "outcome_ptr": 0.3333333333333333,
+    "pairwise_causal_calibration_accuracy": 0.6016754850088184,
+    "pairwise_causal_calibration_confidence": 0.5229362359705314,
+    "pairwise_causal_calibration_ece": 0.11584970123844203,
+    "pairwise_causal_calibration_pairs": 27.933333333333334,
+    "predicted_margin": -0.14503053505926972,
+    "proposal_oracle_success": 0.13333333333333333,
+    "proposal_oracle_utility": 0.6601470321416855,
+    "selected_candidate_index": 4.333333333333333,
+    "selected_success": 0.0,
+    "selected_success_gain_over_base": 0.0,
+    "selected_utility": 0.4771528899669647,
+    "selected_utility_gain_over_base": -0.002456066012382507,
+    "selector_regret": 0.18441228767236073,
+    "success_selector_gap": 0.13333333333333333,
+    "success_support_gap": 0.8666666666666667,
+    "support_gap": 1.3398529678583144,
+    "tau": -0.08708576405217451
+  },
+  "PickCube-v1": {
+    "base_success": 1.0,
+    "base_utility": 2.0,
+    "coverage": 1.0,
+    "execute_generated": 1.0,
+    "fallback_rate": 0.0,
+    "hidden_chart_oracle_success": 1.0,
+    "hidden_chart_oracle_utility": 2.0,
+    "outcome_ptr": 0.0,
+    "pairwise_causal_calibration_accuracy": 0.32142857142857145,
+    "pairwise_causal_calibration_confidence": 0.5472287677691599,
+    "pairwise_causal_calibration_ece": 0.22580019634058843,
+    "pairwise_causal_calibration_pairs": 28.0,
+    "predicted_margin": 0.28070211086543817,
+    "proposal_oracle_success": 0.3333333333333333,
+    "proposal_oracle_utility": 1.021850009759267,
+    "selected_candidate_index": 1.0,
+    "selected_success": 0.0,
+    "selected_success_gain_over_base": -1.0,
+    "selected_utility": 0.26197221378485364,
+    "selected_utility_gain_over_base": -1.7380277862151463,
+    "selector_regret": 0.7598777959744135,
+    "success_selector_gap": 0.3333333333333333,
+    "success_support_gap": 0.6666666666666666,
+    "support_gap": 0.9781499902407328,
+    "tau": -0.08708576405217451
+  },
+  "PullCube-v1": {
+    "base_success": 0.36,
+    "base_utility": 0.8032506120204925,
+    "coverage": 0.13333333333333333,
+    "execute_generated": 0.13333333333333333,
+    "fallback_rate": 0.8666666666666667,
+    "hidden_chart_oracle_success": 0.52,
+    "hidden_chart_oracle_utility": 1.1467878860235214,
+    "outcome_ptr": 0.32,
+    "pairwise_causal_calibration_accuracy": 0.5106143856143858,
+    "pairwise_causal_calibration_confidence": 0.5170252389197862,
+    "pairwise_causal_calibration_ece": 0.149483534146025,
+    "pairwise_causal_calibration_pairs": 24.16,
+    "predicted_margin": -0.27933737647327517,
+    "proposal_oracle_success": 0.17333333333333334,
+    "proposal_oracle_utility": 0.46170220340291657,
+    "selected_candidate_index": 3.2666666666666666,
+    "selected_success": 0.3466666666666667,
+    "selected_success_gain_over_base": -0.013333333333333334,
+    "selected_utility": 0.7870221028725306,
+    "selected_utility_gain_over_base": -0.016228509147961933,
+    "selector_regret": 0.01846348727742831,
+    "success_selector_gap": 0.0,
+    "success_support_gap": 0.3466666666666667,
+    "support_gap": 0.6881442584097386,
+    "tau": -0.08708576405217451
+  },
+  "PushCube-v1": {
+    "base_success": 0.8333333333333334,
+    "base_utility": 1.6734557673335075,
+    "coverage": 0.0,
+    "execute_generated": 0.0,
+    "fallback_rate": 1.0,
+    "hidden_chart_oracle_success": 0.8333333333333334,
+    "hidden_chart_oracle_utility": 1.7015800600250561,
+    "outcome_ptr": 0.16666666666666666,
+    "pairwise_causal_calibration_accuracy": 0.5693706076059017,
+    "pairwise_causal_calibration_confidence": 0.5109774849646357,
+    "pairwise_causal_calibration_ece": 0.1513149797579122,
+    "pairwise_causal_calibration_pairs": 23.833333333333332,
+    "predicted_margin": -0.3951348987851248,
+    "proposal_oracle_success": 0.5555555555555556,
+    "proposal_oracle_utility": 1.1790093589160178,
+    "selected_candidate_index": 2.5555555555555554,
+    "selected_success": 0.8333333333333334,
+    "selected_success_gain_over_base": 0.0,
+    "selected_utility": 1.6734557673335075,
+    "selected_utility_gain_over_base": 0.0,
+    "selector_regret": 0.32654423266649246,
+    "success_selector_gap": 0.16666666666666666,
+    "success_support_gap": 0.3333333333333333,
+    "support_gap": 0.6220440144340197,
+    "tau": -0.08708576405217451
+  },
+  "StackCube-v1": {
+    "base_success": 0.09090909090909091,
+    "base_utility": 0.5003613992170854,
+    "coverage": 0.45454545454545453,
+    "execute_generated": 0.45454545454545453,
+    "fallback_rate": 0.5454545454545454,
+    "hidden_chart_oracle_success": 1.0,
+    "hidden_chart_oracle_utility": 2.0,
+    "outcome_ptr": 0.6060606060606061,
+    "pairwise_causal_calibration_accuracy": 0.5182683982683982,
+    "pairwise_causal_calibration_confidence": 0.527034482028972,
+    "pairwise_causal_calibration_ece": 0.12741592448328593,
+    "pairwise_causal_calibration_pairs": 27.818181818181817,
+    "predicted_margin": -0.07917090287344432,
+    "proposal_oracle_success": 0.18181818181818182,
+    "proposal_oracle_utility": 0.634040683056369,
+    "selected_candidate_index": 2.757575757575758,
+    "selected_success": 0.09090909090909091,
+    "selected_success_gain_over_base": 0.0,
+    "selected_utility": 0.48857860628402594,
+    "selected_utility_gain_over_base": -0.011782792933059462,
+    "selector_regret": 0.18203219771385193,
+    "success_selector_gap": 0.12121212121212122,
+    "success_support_gap": 0.8181818181818182,
+    "support_gap": 1.365959316943631,
+    "tau": -0.08708576405217451
+  }
+}

workspace/runs/ctt_base_context_obs_learned_dominance_basic_perdim_trainmax_train_to_test/split_hash.txt

@@ -0,0 +1,14 @@
+{
+  "calibration_target_index": {
+    "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42",
+    "retrieval_index_allowed": true,
+    "split": "train",
+    "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372"
+  },
+  "eval_target_index": {
+    "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af",
+    "retrieval_index_allowed": false,
+    "split": "test",
+    "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f"
+  }
+}

workspace/runs/ctt_base_context_obs_learned_dominance_basic_perdim_trainmax_train_to_test/table.tex

@@ -0,0 +1,8 @@
+% Auto-generated by scripts/eval_learned_dominance_selector.py
+\begin{tabular}{lrrrrrrrrr}
+\toprule
+Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap & Cal. ECE \\
+\midrule
+144 & 0.2361 & 0.7639 & 0.3333 & 0.3056 & 0.2222 & 0.3611 & 0.5139 & 0.0694 & 0.0103 \\
+\bottomrule
+\end{tabular}

workspace/runs/ctt_base_context_obs_learned_dominance_basic_perdim_trainmax_train_to_test/train.log

@@ -0,0 +1,5 @@
+trained ridge dominance calibrator on calibration measured rows only
+fit_objective=pointwise
+selected_lambda=0.0
+tau=-0.087086
+calibration_input=runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json