diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/command.txt b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..dd18d1098592bc0bd372bc1f401a67f816c38197 --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_learned_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --source-index data/cil_charts_rgb_refs/train/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test --feature-set score_chart_compat --selector-chart-feature-mode base_context_obs --target success_weighted_margin --success-bonus 2.0 --threshold-scope task --k 16 --bootstrap-samples 300 --no-markdown-report diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/config.yaml b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfabeae8a9f08286bdbc68a972265e7d4d8fe0b3 --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/config.yaml @@ -0,0 +1,18 @@ +bootstrap_samples: 300 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: score_chart_compat +fit_objective: pointwise +k: 16 +no_markdown_report: True +out_dir: runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test +pairwise_weight: 1.0 +ridge_lambdas: 0,0.01,0.1,1,10,100 +selector_chart_feature_mode: base_context_obs +source_index: data/cil_charts_rgb_refs/train/index.json +success_bonus: 2.0 +target: success_weighted_margin +threshold_scope: task diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/data_hash.txt b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..e65c158c17e61aaf10422563d2ffdd8922735e6a --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/data_hash.txt @@ -0,0 +1,7 @@ +{ + "calibration_input": "d79e275b896eb98caa128470d4614b76960cab12407faa157c583224be7abe80", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "541d5a928da5f7ee04ca3dea570e04208830dffde431edfe5b5e38b692634045", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36", + "source_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85" +} diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/eval.log b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..ad44bd37e51f82471f7a17d7372cee5fed00ed3f --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/eval.log @@ -0,0 +1,3 @@ +evaluated learned fallback rule on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/git_hash.txt b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/metrics.json b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..1909c5fc2efda51578c0d723a6f48a295c892dd4 --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/metrics.json @@ -0,0 +1,9008 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "calibration_model_selection": { + "base_success": 0.2777777777777778, + "base_utility": 0.7007981554626086, + "coverage": 0.5138888888888888, + "fallback_rate": 0.4861111111111111, + "hidden_chart_oracle_success": 0.6597222222222222, + "hidden_chart_oracle_utility": 1.4150809271054134, + "outcome_ptr": 0.5462962962962963, + "proposal_oracle_success": 0.5370370370370371, + "proposal_oracle_utility": 1.1440672337592803, + "selected_success": 0.35185185185185186, + "selected_success_gain_over_base": 0.07407407407407407, + "selected_utility": 0.8191827951105316, + "selector_regret": 0.41054302532871517, + "success_selector_gap": 0.22453703703703703, + "success_support_gap": 0.17592592592592593, + "support_gap": 0.35465237537627453 + }, + "calibration_summary": { + "base_success": 0.2777777777777778, + "base_utility": 0.7007981554626086, + "coverage": 0.5138888888888888, + "fallback_rate": 0.4861111111111111, + "hidden_chart_oracle_success": 0.6597222222222222, + "hidden_chart_oracle_utility": 1.4150809271054134, + "outcome_ptr": 0.5462962962962963, + "pairwise_causal_calibration_accuracy": 0.5058718205276739, + "pairwise_causal_calibration_confidence": 0.5568104969283495, + "pairwise_causal_calibration_ece": 0.05093867640067546, + "pairwise_causal_calibration_pairs": 37978.0, + "proposal_oracle_success": 0.5370370370370371, + "proposal_oracle_utility": 1.1440672337592803, + "selected_success": 0.35185185185185186, + "selected_success_gain_over_base": 0.07407407407407407, + "selected_utility": 0.8191827951105316, + "selector_regret": 0.41054302532871517, + "success_selector_gap": 0.22453703703703703, + "success_support_gap": 0.17592592592592593, + "support_gap": 0.35465237537627453 + }, + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.2916666666666667, + "base_utility": 0.7418195864423373, + "coverage": 0.5486111111111112, + "fallback_rate": 0.4513888888888889, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.5486111111111112, + "pairwise_causal_calibration_accuracy": 0.5247707897699855, + "pairwise_causal_calibration_confidence": 0.5549377694154455, + "pairwise_causal_calibration_ece": 0.030166979645460227, + "pairwise_causal_calibration_pairs": 12434.0, + "proposal_oracle_success": 0.5694444444444444, + "proposal_oracle_utility": 1.2159999172436073, + "selected_success": 0.3194444444444444, + "selected_success_gain_over_base": 0.027777777777777776, + "selected_utility": 0.7728274115272344, + "selector_regret": 0.45799514941932606, + "success_selector_gap": 0.25, + "success_support_gap": 0.2013888888888889, + "support_gap": 0.3761285081272945 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_mean": [ + 0.0, + 4.602051421289207, + 1.8887871984592688, + -1.985099232970326e-17, + 7.187557603686636, + 0.0663594470046083, + 2.0903225806451613, + 0.7930022866760579, + 1.9819565879721794, + 15.375115207373272, + 0.5, + 0.06635944700460818, + -1.1461792655128946, + -0.8539997540940796, + -0.16156509362744845, + 0.16156509362744845, + 0.5331797235023041, + 0.29217951141881504, + 1.0, + 1.044349800585736, + 1.0509106460665045, + 0.8002031215841607, + 0.5811250768510506, + 2.5434513049028507, + 0.34326541665726357, + 0.8966401857535892, + 1.2182893520781068, + 1.2263923089918751, + 0.786961656617233, + 0.6987682628846764, + 1.0, + 0.9974076898437442, + 0.02008356847617045, + 0.39137378168005876, + 0.3910654018162905, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "candidate_score_rank_fraction", + "candidate_score_softmax_prob", + "candidate_score_gap_to_best", + "candidate_score_gap_to_second_best", + "candidate_score_gap_to_prev_higher", + "candidate_score_gap_to_next_lower", + "candidate_score_percentile", + "candidate_score_top_margin", + "source_chart_feature_found", + "target_chart_feature_norm", + "source_chart_feature_norm", + "target_source_chart_cosine", + "target_source_chart_rms", + "target_source_chart_linf", + "target_source_chart_absmean", + "target_source_chart_dot_per_dim", + "target_base_action_norm", + "source_base_action_norm", + "target_source_base_cosine", + "target_source_base_rms", + "target_source_obs_available", + "target_source_obs_cosine", + "target_source_obs_rms", + "target_obs_norm", + "source_obs_norm", + "target_source_obj_available", + "target_source_obj_cosine", + "target_source_obj_rms", + "target_obj_norm", + "source_obj_norm" + ], + "feature_set": "score_chart_compat", + "feature_std": [ + 1.0, + 2.5232884736710055, + 2.2444575887945213, + 0.9999992484766103, + 4.542613580634851, + 0.24891036261589178, + 1.9642314545113713, + 0.37078681697003224, + 0.7171840796063713, + 1.7485561930736828, + 0.30856036102426265, + 0.04867152161828175, + 0.9290082113338436, + 0.8520345362910837, + 0.21685697548301888, + 0.21685697548301888, + 0.2880797360439561, + 0.30393911054414136, + 1e-06, + 0.14319094131293478, + 0.14491309378768996, + 0.19298897178584146, + 0.32495613500512616, + 1.3413619446954763, + 0.2065176097090355, + 0.3376678168884184, + 0.1766334065498482, + 0.1790096989966568, + 0.20687067948130008, + 0.3907071033474156, + 1e-06, + 0.005866370796541541, + 0.02376663855951178, + 0.015525196749693172, + 0.017227249939028074, + 1e-06, + 1e-06, + 1e-06, + 1e-06, + 1e-06 + ], + "fit_design": { + "fit_objective": "pointwise", + "num_candidate_rows": 6510, + "num_fit_rows": 6510, + "num_pairwise_rows": 0, + "num_pointwise_rows": 6510, + "pairwise_weight": 0.0 + }, + "fit_objective": "pointwise", + "k": 16, + "num_calibration_candidates": 6510, + "num_calibration_rows": 432, + "num_eval_candidates": 2115, + "num_eval_rows": 144, + "pairwise_causal_calibration": { + "calibration": { + "accuracy": 0.5058718205276739, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.02938225092245028, + "accuracy": 0.5073257124456609, + "confidence": 0.5367079633681112, + "count": 31055, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.13873592791765804, + "accuracy": 0.49385749385749383, + "confidence": 0.6325934217751519, + "count": 6105, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.15430795591393243, + "accuracy": 0.5841726618705037, + "confidence": 0.7384806177844361, + "count": 695, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.5458597609917754, + "accuracy": 0.2975206611570248, + "confidence": 0.8433804221488002, + "count": 121, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.9036747853344398, + "accuracy": 0.0, + "confidence": 0.9036747853344398, + "count": 2, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.05093867640067546, + "mean_confidence": 0.5568104969283495, + "n_bins": 10, + "num_pairs": 37978, + "num_rows": 432 + }, + "eval": { + "accuracy": 0.5247707897699855, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.009271938591821294, + "accuracy": 0.5264476346468832, + "confidence": 0.5357195732387044, + "count": 10379, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.11527287642897033, + "accuracy": 0.5194585448392555, + "confidence": 0.6347314212682258, + "count": 1773, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.28892814277768075, + "accuracy": 0.44329896907216493, + "confidence": 0.7322271118498457, + "count": 194, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.20945346204210047, + "accuracy": 0.6136363636363636, + "confidence": 0.8230898256784641, + "count": 88, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.030166979645460227, + "mean_confidence": 0.5549377694154455, + "n_bins": 10, + "num_pairs": 12434, + "num_rows": 144 + } + }, + "pairwise_weight": 1.0, + "report_type": "learned_dominance_selector_eval", + "ridge_lambdas": [ + 0.0, + 0.01, + 0.1, + 1.0, + 10.0, + 100.0 + ], + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.48739495798319327, + "pairwise_causal_calibration_confidence": 0.533600718072752, + "pairwise_causal_calibration_ece": 0.04620576008955871, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.107133532936841, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.713559627532959, + "selected_utility_gain_over_base": -0.01004570722579956, + "selector_regret": 1.286440372467041, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5494505494505495, + "pairwise_causal_calibration_confidence": 0.5728850640530763, + "pairwise_causal_calibration_ece": 0.1199981805489268, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": 0.7747416665744656, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "seed": "307068879/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6134453781512605, + "pairwise_causal_calibration_confidence": 0.5462334122382229, + "pairwise_causal_calibration_ece": 0.06721196591303755, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.8315109279797959, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4857142857142857, + "pairwise_causal_calibration_confidence": 0.594216772727526, + "pairwise_causal_calibration_ece": 0.11412216411454103, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 1.589765155203597, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.707331120967865, + "selected_utility_gain_over_base": -0.018370509147644043, + "selector_regret": 1.292668879032135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.38181818181818183, + "pairwise_causal_calibration_confidence": 0.5664245191730715, + "pairwise_causal_calibration_ece": 0.19522652344582997, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": 0.6069194020559678, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "441046872/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.3000448346138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.550550217481388, + "pairwise_causal_calibration_ece": 0.11611644918527875, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.3705665707855397, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "seed": "1522272686/train0", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "task_id": "PickCube-v1", + "tau": 1.2894892762347252, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.6807136864797709, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.49572649572649574, + "pairwise_causal_calibration_confidence": 0.5264649509158664, + "pairwise_causal_calibration_ece": 0.030738455189370684, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": 0.5168928978077507, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8383140712976456, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.523550311048989, + "pairwise_causal_calibration_ece": 0.14311635561767766, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.06695685840699794, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "seed": "1461069713/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.6566064262371367, + "pairwise_causal_calibration_ece": 0.27387444019257834, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.3361085944561055, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.010183374397456646, + "seed": "626486813/train0", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.005017320159822702, + "selected_utility_gain_over_base": 0.0030878185061737895, + "selector_regret": 0.0051660542376339436, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24323733616620302, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.26666666666666666, + "pairwise_causal_calibration_confidence": 0.5291897348449885, + "pairwise_causal_calibration_ece": 0.2625230681783219, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.8451839925989519, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15456630289554596, + "seed": "2054899126/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.12878061830997467, + "selected_utility_gain_over_base": -1.8712193816900253, + "selector_regret": 0.02578568458557129, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.845433697104454, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8235294117647058, + "pairwise_causal_calibration_confidence": 0.5940991028072462, + "pairwise_causal_calibration_ece": 0.22943030895745967, + "pairwise_causal_calibration_pairs": 17.0, + "predicted_margin": 0.7871393331842744, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5787709662074795, + "pairwise_causal_calibration_ece": 0.16210429954081287, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.4206163793097437, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15051491558551788, + "seed": "767747424/train0", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07352641969919205, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08458571135997772, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7333333333333333, + "pairwise_causal_calibration_confidence": 0.6097360202282281, + "pairwise_causal_calibration_ece": 0.12359731310510547, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.8291893276454425, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14671264588832855, + "seed": "278008524/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05741874501109123, + "selected_utility_gain_over_base": 0.021532438695430756, + "selector_regret": 0.08929390087723732, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08110463619232178, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.41304347826086957, + "pairwise_causal_calibration_confidence": 0.51605426349952, + "pairwise_causal_calibration_ece": 0.10301078523865043, + "pairwise_causal_calibration_pairs": 46.0, + "predicted_margin": 0.798908624848506, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.55, + "pairwise_causal_calibration_confidence": 0.5751237107284101, + "pairwise_causal_calibration_ece": 0.07811172592180064, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.5841434390229071, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07920757681131363, + "seed": "626486813/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.009500582702457905, + "selected_utility_gain_over_base": 0.0012333989143371582, + "selector_regret": 0.06970699410885572, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5491820139582639, + "pairwise_causal_calibration_ece": 0.11748465270840278, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.5904121339165793, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.001021603704430163, + "selected_utility_gain_over_base": -1.9989783962955698, + "selector_regret": 1.9989783962955698, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.36904761904761907, + "pairwise_causal_calibration_confidence": 0.5472546704459165, + "pairwise_causal_calibration_ece": 0.17820705139829746, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.9073249211388359, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09919407218694687, + "selected_utility_gain_over_base": -0.0825214758515358, + "selector_regret": 1.9008059278130531, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2833333333333333, + "pairwise_causal_calibration_confidence": 0.5448471499959241, + "pairwise_causal_calibration_ece": 0.2615138166625908, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.5415586302800286, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "seed": "141984615/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6333333333333333, + "pairwise_causal_calibration_confidence": 0.5341284371266161, + "pairwise_causal_calibration_ece": 0.09920489620671744, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.9654271375816255, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21658658981323242, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11457405239343643, + "selected_utility_gain_over_base": -0.02813490480184555, + "selector_regret": 0.10201253741979599, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7834134101867676, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.425, + "pairwise_causal_calibration_confidence": 0.540894445846163, + "pairwise_causal_calibration_ece": 0.11589444584616293, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.0758545824030454, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "seed": "636680137/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.056092143058776855, + "selected_utility_gain_over_base": -0.027042903006076813, + "selector_regret": 0.0321691632270813, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.38333333333333336, + "pairwise_causal_calibration_confidence": 0.5826379337146643, + "pairwise_causal_calibration_ece": 0.19930460038133097, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.6551650928365002, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.019201835617423058, + "seed": "767747424/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.007447937969118357, + "selected_utility_gain_over_base": -0.046675005462020636, + "selector_regret": 0.0117538976483047, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17790568806231022, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5866666666666667, + "pairwise_causal_calibration_confidence": 0.5192841401886225, + "pairwise_causal_calibration_ece": 0.06738252647804421, + "pairwise_causal_calibration_pairs": 75.0, + "predicted_margin": 0.13235098862844796, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.27586206896551724, + "pairwise_causal_calibration_confidence": 0.5879562438719218, + "pairwise_causal_calibration_ece": 0.31209417490640445, + "pairwise_causal_calibration_pairs": 29.0, + "predicted_margin": 1.2008765491740636, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6166666666666667, + "pairwise_causal_calibration_confidence": 0.5409741214707926, + "pairwise_causal_calibration_ece": 0.07569254519587415, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.3177872369672483, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.041394393891096115, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.041394393891096115, + "selected_utility_gain_over_base": -0.05160309746861458, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18259509280323982, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 1.1837230041552602, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5083333333333333, + "pairwise_causal_calibration_confidence": 0.5322967641018285, + "pairwise_causal_calibration_ece": 0.046545342019779196, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.7403769985406177, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "seed": "747260437/train0", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07023122161626816, + "selected_utility_gain_over_base": -1.9297687783837318, + "selector_regret": 0.0381644144654274, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5050505050505051, + "pairwise_causal_calibration_confidence": 0.5720678596122971, + "pairwise_causal_calibration_ece": 0.14539609879445076, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": -0.27774835711899776, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04958367347717285, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9504163265228271, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.43333333333333335, + "pairwise_causal_calibration_confidence": 0.5370476153908349, + "pairwise_causal_calibration_ece": 0.10371428205750155, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.332576441483642, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "seed": "1549324805/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.0347811970859766, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.575, + "pairwise_causal_calibration_confidence": 0.5678773715560055, + "pairwise_causal_calibration_ece": 0.0719433679479887, + "pairwise_causal_calibration_pairs": 40.0, + "predicted_margin": 0.7418095681624295, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06306678056716919, + "selected_utility_gain_over_base": -0.008848525583744049, + "selector_regret": 1.9369332194328308, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4880952380952381, + "pairwise_causal_calibration_confidence": 0.5328695070998236, + "pairwise_causal_calibration_ece": 0.04477426900458553, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.11786003398422906, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6551724137931034, + "pairwise_causal_calibration_confidence": 0.5624630563402808, + "pairwise_causal_calibration_ece": 0.09270935745282255, + "pairwise_causal_calibration_pairs": 29.0, + "predicted_margin": 1.7706684649919293, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5666666666666667, + "pairwise_causal_calibration_confidence": 0.5461265642044052, + "pairwise_causal_calibration_ece": 0.020540102462261362, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.8003040878491947, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "seed": "1324642411/train0", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12164360284805298, + "selected_utility_gain_over_base": 0.015173077583312988, + "selector_regret": 0.02241644263267517, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2818181818181818, + "pairwise_causal_calibration_confidence": 0.5215209331218725, + "pairwise_causal_calibration_ece": 0.23970275130369073, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 1.1652542261433299, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.5514987071348044, + "pairwise_causal_calibration_ece": 0.09695325258934998, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.7394668700396011, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6386554621848739, + "pairwise_causal_calibration_confidence": 0.5365694240669033, + "pairwise_causal_calibration_ece": 0.10561970831698192, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.7250273458520244, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11310504376888275, + "selected_utility_gain_over_base": 0.11310482025146484, + "selector_regret": 1.8868949562311172, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8282828282828283, + "pairwise_causal_calibration_confidence": 0.5709539167236203, + "pairwise_causal_calibration_ece": 0.2573289115592079, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 2.2046095259547753, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.5639694431421344, + "pairwise_causal_calibration_ece": 0.130957329325739, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.39584763737906137, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "seed": "2061635888/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5833333333333334, + "pairwise_causal_calibration_confidence": 0.5958635862373126, + "pairwise_causal_calibration_ece": 0.10068633489228643, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.5292851601669687, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "seed": "655884605/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5542145473380933, + "pairwise_causal_calibration_ece": 0.08992883305237898, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.028473507842246004, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "seed": "1113371261/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7350427350427351, + "pairwise_causal_calibration_confidence": 0.575799443033985, + "pairwise_causal_calibration_ece": 0.15924329200874998, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": 1.3204376505810713, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train0", + "selected_candidate_index": 9, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.17586350440979, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.5523718802888111, + "pairwise_causal_calibration_ece": 0.10792743584436665, + "pairwise_causal_calibration_pairs": 90.0, + "predicted_margin": 0.6801689818523341, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6302521008403361, + "pairwise_causal_calibration_confidence": 0.5426955700305087, + "pairwise_causal_calibration_ece": 0.08755653080982741, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.1272780451605109, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "211245842/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8592818975448608, + "selected_utility_gain_over_base": 0.6683734804391861, + "selector_regret": 1.1407181024551392, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5428571428571428, + "pairwise_causal_calibration_confidence": 0.5504892981594165, + "pairwise_causal_calibration_ece": 0.007632155302273683, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.8265443858364611, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "seed": "1151410346/train0", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5765433563593303, + "pairwise_causal_calibration_ece": 0.1479719277879018, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3146091278603853, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "seed": "1324133122/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.38095238095238093, + "pairwise_causal_calibration_confidence": 0.5278746131288938, + "pairwise_causal_calibration_ece": 0.1469222321765129, + "pairwise_causal_calibration_pairs": 42.0, + "predicted_margin": 1.1356123561511042, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train0", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.65, + "pairwise_causal_calibration_confidence": 0.5493461173759878, + "pairwise_causal_calibration_ece": 0.10951912707521982, + "pairwise_causal_calibration_pairs": 60.0, + "predicted_margin": 1.0435143603129777, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.543208103565328, + "pairwise_causal_calibration_ece": 0.1860652464224708, + "pairwise_causal_calibration_pairs": 70.0, + "predicted_margin": 0.8291022763943762, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6083333333333333, + "pairwise_causal_calibration_confidence": 0.5379143734950628, + "pairwise_causal_calibration_ece": 0.07041895983827043, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.9306280702125403, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7183246612548828, + "selected_utility_gain_over_base": -0.005280673503875732, + "selector_regret": 1.2816753387451172, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5274725274725275, + "pairwise_causal_calibration_confidence": 0.5704103700900957, + "pairwise_causal_calibration_ece": 0.07309450817078264, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": 0.3600826916101192, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "seed": "307068879/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6833333333333333, + "pairwise_causal_calibration_confidence": 0.5414540870914354, + "pairwise_causal_calibration_ece": 0.14187924624189804, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.696723804413939, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train1", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5047619047619047, + "pairwise_causal_calibration_confidence": 0.5978828235074669, + "pairwise_causal_calibration_ece": 0.20716363895507375, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 1.3392733370703795, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.274298369884491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6181818181818182, + "pairwise_causal_calibration_confidence": 0.5652066435973622, + "pairwise_causal_calibration_ece": 0.09055210003018804, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": 0.1629836409186699, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "seed": "441046872/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5518834340111771, + "pairwise_causal_calibration_ece": 0.13521676734451044, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.8136424781392986, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "seed": "1522272686/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "task_id": "PickCube-v1", + "tau": 1.2894892762347252, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.7412627528047919, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4369747899159664, + "pairwise_causal_calibration_confidence": 0.5350494099395171, + "pairwise_causal_calibration_ece": 0.09807462002355061, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.3100122906270366, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8383140712976456, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3333333333333333, + "pairwise_causal_calibration_confidence": 0.5236824574508954, + "pairwise_causal_calibration_ece": 0.19034912411756205, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.24721792540099097, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "seed": "1461069713/train1", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.26666666666666666, + "pairwise_causal_calibration_confidence": 0.6443129578291337, + "pairwise_causal_calibration_ece": 0.37764629116246684, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.1814917850355002, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.009895960800349712, + "seed": "626486813/train1", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00289076566696167, + "selected_utility_gain_over_base": 0.000961264013312757, + "selector_regret": 0.0070051951333880424, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24352474976330996, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6754385964912281, + "pairwise_causal_calibration_confidence": 0.5263065559429049, + "pairwise_causal_calibration_ece": 0.14913204054832319, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": 0.6246851300643428, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5428571428571428, + "pairwise_causal_calibration_confidence": 0.5924504110795782, + "pairwise_causal_calibration_ece": 0.07756824536937998, + "pairwise_causal_calibration_pairs": 35.0, + "predicted_margin": 0.7852289914727505, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5916666666666667, + "pairwise_causal_calibration_confidence": 0.5516469249954096, + "pairwise_causal_calibration_ece": 0.09795148252740986, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.47195461369487435, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20903043448925018, + "seed": "767747424/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.13204193860292435, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026070192456245422, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5583333333333333, + "pairwise_causal_calibration_confidence": 0.5898531385273096, + "pairwise_causal_calibration_ece": 0.06222000578143905, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.9399097065125667, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07534918934106827, + "seed": "278008524/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01919553615152836, + "selected_utility_gain_over_base": -0.016690770164132118, + "selector_regret": 0.05615365318953991, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15246809273958206, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7692307692307693, + "pairwise_causal_calibration_confidence": 0.5252519047235767, + "pairwise_causal_calibration_ece": 0.24397886450719253, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": 0.8663205932771525, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39166666666666666, + "pairwise_causal_calibration_confidence": 0.5647193558303677, + "pairwise_causal_calibration_ece": 0.17305268916370073, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.7391401538679906, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.061759788542985916, + "seed": "626486813/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.019184648990631104, + "selected_utility_gain_over_base": 0.010917465202510357, + "selector_regret": 0.04257513955235481, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01598426327109337, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7350427350427351, + "pairwise_causal_calibration_confidence": 0.5263645115117457, + "pairwise_causal_calibration_ece": 0.20867822353098942, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": 0.5895406979096317, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.64, + "pairwise_causal_calibration_confidence": 0.549320518336562, + "pairwise_causal_calibration_ece": 0.19650416069860938, + "pairwise_causal_calibration_pairs": 75.0, + "predicted_margin": 0.8486373501760932, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train1", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04792279005050659, + "selected_utility_gain_over_base": -0.13379275798797607, + "selector_regret": 1.9520772099494934, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6083333333333333, + "pairwise_causal_calibration_confidence": 0.5656089960064071, + "pairwise_causal_calibration_ece": 0.14136934515423885, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.8524230828537844, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "seed": "141984615/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.027254482731223106, + "selected_utility_gain_over_base": -0.07557976432144642, + "selector_regret": 0.009756645187735558, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.425, + "pairwise_causal_calibration_confidence": 0.5283042091940678, + "pairwise_causal_calibration_ece": 0.10330420919406785, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.297807510881273, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.625, + "pairwise_causal_calibration_confidence": 0.5301042033378723, + "pairwise_causal_calibration_ece": 0.09489579666212766, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.44432064636845636, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "seed": "636680137/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5166666666666667, + "pairwise_causal_calibration_confidence": 0.597444625657663, + "pairwise_causal_calibration_ece": 0.21983253135176184, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.0400199103730006, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.042781930416822433, + "seed": "767747424/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.010063628666102886, + "selected_utility_gain_over_base": -0.044059314765036106, + "selector_regret": 0.03271830175071955, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15432559326291084, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5386998051668176, + "pairwise_causal_calibration_ece": 0.03869980516681758, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": -0.48662334960024145, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train1", + "selected_candidate_index": 15, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.24074074074074073, + "pairwise_causal_calibration_confidence": 0.6117747139844449, + "pairwise_causal_calibration_ece": 0.37103397324370413, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 1.112157225492381, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07674666494131088, + "selected_utility_gain_over_base": -1.9232533350586891, + "selector_regret": 1.9232533350586891, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.425, + "pairwise_causal_calibration_confidence": 0.5397340717215419, + "pairwise_causal_calibration_ece": 0.11729596961499984, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.7678920642673333, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11150982230901718, + "seed": "750448296/train1", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06182505562901497, + "selected_utility_gain_over_base": -0.031172435730695724, + "selector_regret": 0.04968476668000221, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.11247966438531876, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 1.5233706918468892, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.675, + "pairwise_causal_calibration_confidence": 0.5502411276617256, + "pairwise_causal_calibration_ece": 0.2890079292574854, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.6210780034951271, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "seed": "747260437/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.05181918665766716, + "selected_utility_gain_over_base": -1.9481808133423328, + "selector_regret": 0.011470336467027664, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6086956521739131, + "pairwise_causal_calibration_confidence": 0.5797446400488626, + "pairwise_causal_calibration_ece": 0.07560374307224685, + "pairwise_causal_calibration_pairs": 92.0, + "predicted_margin": -0.22025978406278973, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04958367347717285, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9504163265228271, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.30833333333333335, + "pairwise_causal_calibration_confidence": 0.5437634159530779, + "pairwise_causal_calibration_ece": 0.2354300826197445, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.9503772340326878, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "seed": "1549324805/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0303483996540308, + "selected_utility_gain_over_base": -0.07416643761098385, + "selector_regret": 0.16857311688363552, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6938775510204082, + "pairwise_causal_calibration_confidence": 0.592873970483461, + "pairwise_causal_calibration_ece": 0.10100358053694718, + "pairwise_causal_calibration_pairs": 49.0, + "predicted_margin": 0.7078283118607503, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9280846938490868, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.47692307692307695, + "pairwise_causal_calibration_confidence": 0.5453125451571335, + "pairwise_causal_calibration_ece": 0.06838946823405649, + "pairwise_causal_calibration_pairs": 65.0, + "predicted_margin": 0.1509547219477273, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5333333333333333, + "pairwise_causal_calibration_confidence": 0.5471459072582541, + "pairwise_causal_calibration_ece": 0.3939678976832891, + "pairwise_causal_calibration_pairs": 15.0, + "predicted_margin": 1.808986225138223, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6218487394957983, + "pairwise_causal_calibration_confidence": 0.5434572117117997, + "pairwise_causal_calibration_ece": 0.07839152778399854, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.3876858687209335, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11658626794815063, + "selected_utility_gain_over_base": 0.010115742683410645, + "selector_regret": 1.8834137320518494, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3508771929824561, + "pairwise_causal_calibration_confidence": 0.517198535735028, + "pairwise_causal_calibration_ece": 0.16632134275257193, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": 1.1907212928313995, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10585354268550873, + "selected_utility_gain_over_base": -0.01891392469406128, + "selector_regret": 1.8941464573144913, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4909090909090909, + "pairwise_causal_calibration_confidence": 0.5481788586308872, + "pairwise_causal_calibration_ece": 0.05726976772179637, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.7193134962639322, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5326653211281538, + "pairwise_causal_calibration_ece": 0.038763250300417725, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 1.5703557578982594, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5882352941176471, + "pairwise_causal_calibration_confidence": 0.543456481052637, + "pairwise_causal_calibration_ece": 0.04477881306501008, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.735589926228614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5581364424191588, + "pairwise_causal_calibration_ece": 0.1414697757524921, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.3705725365164993, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "seed": "2061635888/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5277777777777778, + "pairwise_causal_calibration_confidence": 0.6117746577593474, + "pairwise_causal_calibration_ece": 0.228982599451425, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.8854182759065323, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "seed": "655884605/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6498691290616989, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5534020512884718, + "pairwise_causal_calibration_ece": 0.12516937728295682, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.018305243090398104, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "seed": "1113371261/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4090909090909091, + "pairwise_causal_calibration_confidence": 0.5644259196254204, + "pairwise_causal_calibration_ece": 0.15533501053451115, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 1.2385033751871737, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12732218205928802, + "selected_utility_gain_over_base": -0.6968143135309219, + "selector_regret": 1.872677817940712, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.19047619047619047, + "pairwise_causal_calibration_confidence": 0.5517045024701317, + "pairwise_causal_calibration_ece": 0.36122831199394134, + "pairwise_causal_calibration_pairs": 63.0, + "predicted_margin": 0.7807195223473116, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.65, + "pairwise_causal_calibration_confidence": 0.5351083386938548, + "pairwise_causal_calibration_ece": 0.12495817852344455, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.2265101367583866, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "seed": "211245842/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09285585582256317, + "selected_utility_gain_over_base": -0.09805256128311157, + "selector_regret": 0.12010131776332855, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.38095238095238093, + "pairwise_causal_calibration_confidence": 0.5497312288122934, + "pairwise_causal_calibration_ece": 0.16877884785991246, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.9117051526313378, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1151410346/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5703079922305777, + "pairwise_causal_calibration_ece": 0.07659820557682445, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.05209384637709236, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "seed": "1324133122/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5308683756394741, + "pairwise_causal_calibration_ece": 0.004845910074811617, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 1.234980143424118, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train1", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7948717948717948, + "pairwise_causal_calibration_confidence": 0.5593981622164304, + "pairwise_causal_calibration_ece": 0.23547363265536436, + "pairwise_causal_calibration_pairs": 39.0, + "predicted_margin": 0.949568847497014, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6714285714285714, + "pairwise_causal_calibration_confidence": 0.5400358084963361, + "pairwise_causal_calibration_ece": 0.13139276293223526, + "pairwise_causal_calibration_pairs": 70.0, + "predicted_margin": 0.7563954831040236, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7717391304347826, + "pairwise_causal_calibration_confidence": 0.5403873973296214, + "pairwise_causal_calibration_ece": 0.2313517331051612, + "pairwise_causal_calibration_pairs": 92.0, + "predicted_margin": 1.1151401969224932, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train2", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.2763946652412415, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3956043956043956, + "pairwise_causal_calibration_confidence": 0.5723883106472546, + "pairwise_causal_calibration_ece": 0.1767839150428592, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": 0.979654396381198, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "seed": "307068879/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2021283358335495, + "selected_utility_gain_over_base": -0.39796222746372223, + "selector_regret": 0.02808511257171631, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5552324860178256, + "pairwise_causal_calibration_ece": 0.051910371125031565, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.8010166202874374, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.55, + "pairwise_causal_calibration_confidence": 0.6068596136210345, + "pairwise_causal_calibration_ece": 0.0651220182950748, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.4772393565445312, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "seed": "608016290/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6789124011993408, + "selected_utility_gain_over_base": -0.04678922891616821, + "selector_regret": 0.03281521797180176, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.5755444444938004, + "pairwise_causal_calibration_ece": 0.13114432369738743, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": 0.5214657402479244, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "seed": "441046872/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "task_id": "LiftPegUpright-v1", + "tau": 0.87335466614541, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5635063449255251, + "pairwise_causal_calibration_ece": 0.06840618279050634, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.5686852402790281, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "seed": "1522272686/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "task_id": "PickCube-v1", + "tau": 1.2894892762347252, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.7399153467725452, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 15, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6554621848739496, + "pairwise_causal_calibration_confidence": 0.5336391081875892, + "pairwise_causal_calibration_ece": 0.12182307668636036, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.40469624443133256, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8383140712976456, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5083333333333333, + "pairwise_causal_calibration_confidence": 0.5276472177677529, + "pairwise_causal_calibration_ece": 0.019313884434419615, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.2696456255951446, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "seed": "1461069713/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006073415279388428, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7416666666666667, + "pairwise_causal_calibration_confidence": 0.6498155430714577, + "pairwise_causal_calibration_ece": 0.2291282500464284, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.9688416841155103, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03885193541646004, + "seed": "626486813/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.017679253593087196, + "selected_utility_gain_over_base": 0.015749751939438283, + "selector_regret": 0.02117268182337284, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21456877514719963, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5210084033613446, + "pairwise_causal_calibration_confidence": 0.5536973137638624, + "pairwise_causal_calibration_ece": 0.05340737629836459, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.5730407111707803, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2, + "pairwise_causal_calibration_confidence": 0.6012678269199632, + "pairwise_causal_calibration_ece": 0.4012678269199631, + "pairwise_causal_calibration_pairs": 30.0, + "predicted_margin": 0.6365332388231544, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04013325646519661, + "selected_utility_gain_over_base": -1.9598667435348034, + "selector_regret": 1.9598667435348034, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3416666666666667, + "pairwise_causal_calibration_confidence": 0.5761802746367215, + "pairwise_causal_calibration_ece": 0.23451360797005488, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.24727067758313925, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21886132657527924, + "seed": "767747424/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.1418728306889534, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01623930037021637, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4583333333333333, + "pairwise_causal_calibration_confidence": 0.6068247037197843, + "pairwise_causal_calibration_ece": 0.148491370386451, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.8202350551439472, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2718978226184845, + "seed": "278008524/train2", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04554526135325432, + "selected_utility_gain_over_base": 0.009658955037593842, + "selector_regret": 0.22635256126523018, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.72, + "pairwise_causal_calibration_confidence": 0.5274458415050698, + "pairwise_causal_calibration_ece": 0.19255415849493018, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": 0.8229653113797162, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train2", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5083333333333333, + "pairwise_causal_calibration_confidence": 0.5786084601365713, + "pairwise_causal_calibration_ece": 0.07027512680323779, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.578326404421488, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05094657465815544, + "seed": "626486813/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.042679390870034695, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026797477155923843, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5384615384615384, + "pairwise_causal_calibration_confidence": 0.5293057577168471, + "pairwise_causal_calibration_ece": 0.009155780744691366, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": 0.491258714195548, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7272727272727273, + "pairwise_causal_calibration_confidence": 0.5513841945828967, + "pairwise_causal_calibration_ece": 0.1758885326898305, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.9433176675944015, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.425, + "pairwise_causal_calibration_confidence": 0.5603553061579338, + "pairwise_causal_calibration_ece": 0.1353553061579339, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.3015705513281932, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "seed": "141984615/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.55, + "pairwise_causal_calibration_confidence": 0.5285042510062523, + "pairwise_causal_calibration_ece": 0.021495748993747718, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.8104586454289818, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12231997400522232, + "selected_utility_gain_over_base": -0.020388983190059662, + "selector_regret": 1.8776800259947777, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.475, + "pairwise_causal_calibration_confidence": 0.5238615170185813, + "pairwise_causal_calibration_ece": 0.048861517018581324, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.5548863156073045, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "seed": "636680137/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5416666666666666, + "pairwise_causal_calibration_confidence": 0.6039307444274978, + "pairwise_causal_calibration_ece": 0.06615831252348275, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.1187950154023634, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04980705305933952, + "seed": "767747424/train2", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0314757414162159, + "selected_utility_gain_over_base": -0.022647202014923096, + "selector_regret": 0.018331311643123627, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14730047062039375, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7023809523809523, + "pairwise_causal_calibration_confidence": 0.5315351438061215, + "pairwise_causal_calibration_ece": 0.17084580857483078, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.36262329065526533, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train2", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.40404040404040403, + "pairwise_causal_calibration_confidence": 0.5954970669139762, + "pairwise_causal_calibration_ece": 0.19145666287357221, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 1.2364955407573137, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.48333333333333334, + "pairwise_causal_calibration_confidence": 0.5278506560476054, + "pairwise_causal_calibration_ece": 0.05093124656435263, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.776315255538415, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.051346004009246826, + "seed": "750448296/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04953896999359131, + "selected_utility_gain_over_base": -0.043458521366119385, + "selector_regret": 0.0018070340156555176, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1726434826850891, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 1.5690353615664272, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7083333333333334, + "pairwise_causal_calibration_confidence": 0.5279460992108118, + "pairwise_causal_calibration_ece": 0.1803872341225216, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.9133990308397049, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "seed": "747260437/train2", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.0880771279335022, + "selected_utility_gain_over_base": -1.9119228720664978, + "selector_regret": 0.03702475130558014, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5959595959595959, + "pairwise_causal_calibration_confidence": 0.5820771575937359, + "pairwise_causal_calibration_ece": 0.04630465959545786, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.13031225474580455, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04958367347717285, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9504163265228271, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4583333333333333, + "pairwise_causal_calibration_confidence": 0.540090634882582, + "pairwise_causal_calibration_ece": 0.08175730154924889, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.4498268017490537, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "seed": "1549324805/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04690708592534065, + "selected_utility_gain_over_base": -0.057607751339673996, + "selector_regret": 0.06619475409388542, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4, + "pairwise_causal_calibration_confidence": 0.5677066182834786, + "pairwise_causal_calibration_ece": 0.16770661828347844, + "pairwise_causal_calibration_pairs": 40.0, + "predicted_margin": 0.714483922998961, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0730673298239708, + "selected_utility_gain_over_base": 0.0011520236730575562, + "selector_regret": 1.9269326701760292, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5565463429998946, + "pairwise_causal_calibration_ece": 0.13987967633322793, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.07483677485698936, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5825884328057473, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5185185185185185, + "pairwise_causal_calibration_confidence": 0.5475473443233374, + "pairwise_causal_calibration_ece": 0.17413687717265497, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 1.7310319203842686, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.509090909090909, + "pairwise_causal_calibration_confidence": 0.5436846632267275, + "pairwise_causal_calibration_ece": 0.03459375413581838, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.7578621304129007, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.89352947473526, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.43636363636363634, + "pairwise_causal_calibration_confidence": 0.5206471195621108, + "pairwise_causal_calibration_ece": 0.08428348319847451, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 1.0616754451066321, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1341685652732849, + "selected_utility_gain_over_base": 0.009401097893714905, + "selector_regret": 1.865831434726715, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5450693806396976, + "pairwise_causal_calibration_ece": 0.0620734765031595, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.7862881124399304, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6134453781512605, + "pairwise_causal_calibration_confidence": 0.5403636942457117, + "pairwise_causal_calibration_ece": 0.14212516552457483, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.3794654730008236, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "seed": "509430392/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5370429153145398, + "pairwise_causal_calibration_ece": 0.03438565611403155, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.359251321625822, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21894000470638275, + "selected_utility_gain_over_base": 0.1319059282541275, + "selector_regret": 1.7810599952936172, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.8468573198810986, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.5794706736680671, + "pairwise_causal_calibration_ece": 0.13502622922362276, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.345599063353308, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "seed": "2061635888/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4722222222222222, + "pairwise_causal_calibration_confidence": 0.5899703889806722, + "pairwise_causal_calibration_ece": 0.19094254082377798, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.7967317370820184, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "seed": "655884605/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5590997153025572, + "pairwise_causal_calibration_ece": 0.11947171326887146, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.16653094820357445, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "seed": "1113371261/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5644061750173082, + "pairwise_causal_calibration_ece": 0.13583474644587978, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.1890371636445307, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13359522819519043, + "selected_utility_gain_over_base": -0.6905412673950195, + "selector_regret": 1.8664047718048096, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5444444444444444, + "pairwise_causal_calibration_confidence": 0.5506485429344871, + "pairwise_causal_calibration_ece": 0.024528598759442233, + "pairwise_causal_calibration_pairs": 90.0, + "predicted_margin": 0.7965612115773528, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train2", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.525, + "pairwise_causal_calibration_confidence": 0.534871348160612, + "pairwise_causal_calibration_ece": 0.009871348160611883, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.1256343724003985, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "seed": "211245842/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13431161642074585, + "selected_utility_gain_over_base": -0.056596800684928894, + "selector_regret": 0.09957115352153778, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3904761904761905, + "pairwise_causal_calibration_confidence": 0.5495148445011372, + "pairwise_causal_calibration_ece": 0.15903865402494677, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.8848141434142904, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "seed": "1151410346/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5728588815635123, + "pairwise_causal_calibration_ece": 0.21571602442065527, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.31434308061981353, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "seed": "1324133122/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.5331361711412006, + "pairwise_causal_calibration_ece": 0.08869172669675614, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 1.3518684532465013, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8125, + "selected_utility_gain_over_base": -1.1875, + "selector_regret": 1.1875, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4880952380952381, + "pairwise_causal_calibration_confidence": 0.5532244830443317, + "pairwise_causal_calibration_ece": 0.0651292449490937, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.9787258921638226, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train2", + "selected_candidate_index": 9, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3815789473684211, + "pairwise_causal_calibration_confidence": 0.5298970718699219, + "pairwise_causal_calibration_ece": 0.14831812450150075, + "pairwise_causal_calibration_pairs": 76.0, + "predicted_margin": 0.8419564487563823, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train2", + "selected_candidate_index": 13, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 1.0718654827809488, + "train_seed": "2" + } + ], + "schema_version": 1, + "selected_lambda": 10.0, + "selector_calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "selector_chart_feature_mode": "base_context_obs", + "selector_eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "selector_eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "selector_source_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "success_bonus": 2.0, + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38958333333333334, + "low": 0.22083333333333333, + "mean": 0.30625, + "n": 120, + "num_samples": 300 + }, + "mean": 0.30625, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3038787878787878, + "low": 0.09454545454545454, + "mean": 0.19987878787878788, + "n": 5, + "num_samples": 300 + }, + "mean": 0.19987878787878788, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.3680555555555556, + "low": 0.2152777777777778, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9549208872815751, + "low": 0.6291768466153169, + "mean": 0.7901180622977944, + "n": 120, + "num_samples": 300 + }, + "mean": 0.7901180622977944, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.12476746737957001, + "1790188787/train1": 0.12476746737957001, + "1790188787/train2": 0.12476746737957001, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 2.2351741790771484e-07, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.00509834272088483, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.11759314934412639, + "750448296/train1": 0.11759314934412639, + "750448296/train2": 0.11759314934412639, + "767747424/train0": 0.5731993420049548, + "767747424/train1": 0.5731993420049548, + "767747424/train2": 0.5731993420049548 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.879990507716313, + "low": 0.5455794048082704, + "mean": 0.695559797174015, + "n": 5, + "num_samples": 300 + }, + "mean": 0.695559797174015, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7751704612234608, + "PushCube-v1": 0.3924041787783305, + "StackCube-v1": 0.7763947045261209 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.8711436447313948, + "low": 0.6016125785051069, + "mean": 0.7418195864423373, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6326388888888889, + "low": 0.46388888888888885, + "mean": 0.545138888888889, + "n": 120, + "num_samples": 300 + }, + "mean": 0.545138888888889, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6906666666666667, + "low": 0.18521212121212122, + "mean": 0.43410101010101004, + "n": 5, + "num_samples": 300 + }, + "mean": 0.43410101010101004, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.6533333333333333, + "PushCube-v1": 0.7777777777777778, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.6319444444444444, + "low": 0.4722222222222222, + "mean": 0.5486111111111112, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6326388888888889, + "low": 0.46388888888888885, + "mean": 0.545138888888889, + "n": 120, + "num_samples": 300 + }, + "mean": 0.545138888888889, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6906666666666667, + "low": 0.18521212121212122, + "mean": 0.43410101010101004, + "n": 5, + "num_samples": 300 + }, + "mean": 0.43410101010101004, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.6533333333333333, + "PushCube-v1": 0.7777777777777778, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.6319444444444444, + "low": 0.4722222222222222, + "mean": 0.5486111111111112, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5361111111111112, + "low": 0.3673611111111111, + "mean": 0.4548611111111111, + "n": 120, + "num_samples": 300 + }, + "mean": 0.4548611111111111, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 1.0, + "141984615/train1": 0.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.5, + "636680137/train0": 0.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8147878787878788, + "low": 0.30933333333333335, + "mean": 0.5658989898989899, + "n": 5, + "num_samples": 300 + }, + "mean": 0.5658989898989899, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 1.0, + "PullCube-v1": 0.3466666666666667, + "PushCube-v1": 0.2222222222222222, + "StackCube-v1": 0.7272727272727273 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.5277777777777778, + "low": 0.3680555555555556, + "mean": 0.4513888888888889, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8597222222222223, + "low": 0.7097222222222221, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 300 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 300 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7487430048278638, + "low": 1.487454852130678, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 300 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 300 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.6414945029343169, + "low": 1.3965875044878986, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6180555555555555, + "low": 0.44861111111111107, + "mean": 0.5319444444444444, + "n": 120, + "num_samples": 300 + }, + "mean": 0.5319444444444444, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 1.0, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7151515151515151, + "low": 0.23393939393939395, + "mean": 0.49393939393939396, + "n": 5, + "num_samples": 300 + }, + "mean": 0.49393939393939396, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4666666666666667, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.6363636363636364 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.6388888888888888, + "low": 0.4722222222222222, + "mean": 0.5486111111111112, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "pairwise_causal_calibration_accuracy": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5448702331354158, + "low": 0.49797706822658805, + "mean": 0.5231682660617707, + "n": 117, + "num_samples": 300 + }, + "mean": 0.5231682660617707, + "per_seed": { + "1073161778/train0": 0.6134453781512605, + "1073161778/train1": 0.6833333333333333, + "1073161778/train2": 0.6071428571428571, + "1113371261/train0": 0.4642857142857143, + "1113371261/train1": 0.6785714285714286, + "1113371261/train2": 0.6785714285714286, + "1151410346/train0": 0.5428571428571428, + "1151410346/train1": 0.38095238095238093, + "1151410346/train2": 0.3904761904761905, + "1261930456/train0": 0.45454545454545453, + "1261930456/train1": 0.4909090909090909, + "1261930456/train2": 0.6071428571428571, + "1324133122/train0": 0.42857142857142855, + "1324133122/train1": 0.5714285714285714, + "1324133122/train2": 0.35714285714285715, + "1324642411/train0": 0.5666666666666667, + "1324642411/train1": 0.6218487394957983, + "1324642411/train2": 0.509090909090909, + "1375492786/train0": 0.5866666666666667, + "1375492786/train1": 0.5, + "1375492786/train2": 0.7023809523809523, + "141984615/train0": 0.2833333333333333, + "141984615/train1": 0.6083333333333333, + "141984615/train2": 0.425, + "1461069713/train0": 0.6666666666666666, + "1461069713/train1": 0.3333333333333333, + "1461069713/train2": 0.5083333333333333, + "1480856810/train0": 0.8282828282828283, + "1480856810/train1": 0.5882352941176471, + "1480856810/train2": 0.5714285714285714, + "1509980664/train0": 0.4444444444444444, + "1509980664/train1": 0.19047619047619047, + "1509980664/train2": 0.5444444444444444, + "1522272686/train0": 0.6666666666666666, + "1522272686/train1": 0.4166666666666667, + "1522272686/train2": 0.5, + "1549324805/train0": 0.43333333333333335, + "1549324805/train1": 0.30833333333333335, + "1549324805/train2": 0.4583333333333333, + "1578990937/train0": 0.7350427350427351, + "1578990937/train1": 0.4090909090909091, + "1578990937/train2": 0.42857142857142855, + "1790188787/train0": 0.2818181818181818, + "1790188787/train1": 0.3508771929824561, + "1790188787/train2": 0.43636363636363634, + "1822210471/train0": 0.575, + "1822210471/train1": 0.6938775510204082, + "1822210471/train2": 0.4, + "1845489013/train0": 0.41304347826086957, + "1845489013/train1": 0.7692307692307693, + "1845489013/train2": 0.72, + "1999022574/train0": 0.35714285714285715, + "1999022574/train1": 0.6714285714285714, + "1999022574/train2": 0.3815789473684211, + "2012211287/train0": 0.48739495798319327, + "2012211287/train1": 0.6083333333333333, + "2012211287/train2": 0.7717391304347826, + "2038594191/train0": 0.27586206896551724, + "2038594191/train1": 0.24074074074074073, + "2038594191/train2": 0.40404040404040403, + "2054899126/train0": 0.26666666666666666, + "2054899126/train1": 0.6754385964912281, + "2054899126/train2": 0.5210084033613446, + "2061635888/train0": 0.4444444444444444, + "2061635888/train1": 0.4166666666666667, + "2061635888/train2": 0.4444444444444444, + "211245842/train0": 0.6302521008403361, + "211245842/train1": 0.65, + "211245842/train2": 0.525, + "278008524/train0": 0.6191919191919192, + "278008524/train1": 0.5835144927536232, + "278008524/train2": 0.5271464646464646, + "307068879/train0": 0.5494505494505495, + "307068879/train1": 0.5274725274725275, + "307068879/train2": 0.3956043956043956, + "412102563/train0": 0.65, + "412102563/train1": 0.7948717948717948, + "412102563/train2": 0.4880952380952381, + "440577711/train0": 0.6551724137931034, + "440577711/train1": 0.5333333333333333, + "440577711/train2": 0.5185185185185185, + "441046872/train0": 0.38181818181818183, + "441046872/train1": 0.6181818181818182, + "441046872/train2": 0.45454545454545453, + "509430392/train0": 0.6386554621848739, + "509430392/train1": 0.5714285714285714, + "509430392/train2": 0.6134453781512605, + "585904130/train0": 0.36904761904761907, + "585904130/train1": 0.64, + "585904130/train2": 0.7272727272727273, + "608016290/train0": 0.4857142857142857, + "608016290/train1": 0.5047619047619047, + "608016290/train2": 0.55, + "626486813/train0": 0.65, + "626486813/train1": 0.32916666666666666, + "626486813/train2": 0.625, + "636680137/train0": 0.425, + "636680137/train1": 0.625, + "636680137/train2": 0.475, + "655884605/train0": 0.5833333333333334, + "655884605/train1": 0.5277777777777778, + "655884605/train2": 0.4722222222222222, + "672328262/train0": 0.8235294117647058, + "672328262/train1": 0.5428571428571428, + "672328262/train2": 0.2, + "701667486/train0": 0.38095238095238093, + "701667486/train1": 0.5357142857142857, + "701667486/train2": 0.4444444444444444, + "747260437/train0": 0.5083333333333333, + "747260437/train1": 0.675, + "747260437/train2": 0.7083333333333334, + "750448296/train0": 0.5793650793650794, + "750448296/train1": 0.4423076923076923, + "750448296/train2": 0.4833333333333334, + "767747424/train0": 0.49059829059829063, + "767747424/train1": 0.5700877145730087, + "767747424/train2": 0.5193142641672053 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5431888679749804, + "low": 0.5159382160536211, + "mean": 0.5299078728020017, + "n": 5, + "num_samples": 300 + }, + "mean": 0.5299078728020017, + "per_task": { + "LiftPegUpright-v1": 0.5492625405285252, + "PickCube-v1": 0.5277777777777778, + "PullCube-v1": 0.5216143238307843, + "PushCube-v1": 0.5465423944585421, + "StackCube-v1": 0.5043423274143793 + } + }, + "metric": "pairwise_causal_calibration_accuracy", + "micro": { + "confidence": 0.95, + "high": 0.5451467233807504, + "low": 0.5030967497956031, + "mean": 0.5238747803660404, + "n": 138, + "num_samples": 300 + }, + "n": 138 + }, + "pairwise_causal_calibration_confidence": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5604558890777185, + "low": 0.550776431898817, + "mean": 0.5552154770559953, + "n": 117, + "num_samples": 300 + }, + "mean": 0.5552154770559953, + "per_seed": { + "1073161778/train0": 0.5462334122382229, + "1073161778/train1": 0.5414540870914354, + "1073161778/train2": 0.5552324860178256, + "1113371261/train0": 0.5542145473380933, + "1113371261/train1": 0.5534020512884718, + "1113371261/train2": 0.5590997153025572, + "1151410346/train0": 0.5504892981594165, + "1151410346/train1": 0.5497312288122934, + "1151410346/train2": 0.5495148445011372, + "1261930456/train0": 0.5514987071348044, + "1261930456/train1": 0.5481788586308872, + "1261930456/train2": 0.5450693806396976, + "1324133122/train0": 0.5765433563593303, + "1324133122/train1": 0.5703079922305777, + "1324133122/train2": 0.5728588815635123, + "1324642411/train0": 0.5461265642044052, + "1324642411/train1": 0.5434572117117997, + "1324642411/train2": 0.5436846632267275, + "1375492786/train0": 0.5192841401886225, + "1375492786/train1": 0.5386998051668176, + "1375492786/train2": 0.5315351438061215, + "141984615/train0": 0.5448471499959241, + "141984615/train1": 0.5656089960064071, + "141984615/train2": 0.5603553061579338, + "1461069713/train0": 0.523550311048989, + "1461069713/train1": 0.5236824574508954, + "1461069713/train2": 0.5276472177677529, + "1480856810/train0": 0.5709539167236203, + "1480856810/train1": 0.543456481052637, + "1480856810/train2": 0.5370429153145398, + "1509980664/train0": 0.5523718802888111, + "1509980664/train1": 0.5517045024701317, + "1509980664/train2": 0.5506485429344871, + "1522272686/train0": 0.550550217481388, + "1522272686/train1": 0.5518834340111771, + "1522272686/train2": 0.5635063449255251, + "1549324805/train0": 0.5370476153908349, + "1549324805/train1": 0.5437634159530779, + "1549324805/train2": 0.540090634882582, + "1578990937/train0": 0.575799443033985, + "1578990937/train1": 0.5644259196254204, + "1578990937/train2": 0.5644061750173082, + "1790188787/train0": 0.5215209331218725, + "1790188787/train1": 0.517198535735028, + "1790188787/train2": 0.5206471195621108, + "1822210471/train0": 0.5678773715560055, + "1822210471/train1": 0.592873970483461, + "1822210471/train2": 0.5677066182834786, + "1845489013/train0": 0.51605426349952, + "1845489013/train1": 0.5252519047235767, + "1845489013/train2": 0.5274458415050698, + "1999022574/train0": 0.543208103565328, + "1999022574/train1": 0.5400358084963361, + "1999022574/train2": 0.5298970718699219, + "2012211287/train0": 0.533600718072752, + "2012211287/train1": 0.5379143734950628, + "2012211287/train2": 0.5403873973296214, + "2038594191/train0": 0.5879562438719218, + "2038594191/train1": 0.6117747139844449, + "2038594191/train2": 0.5954970669139762, + "2054899126/train0": 0.5291897348449885, + "2054899126/train1": 0.5263065559429049, + "2054899126/train2": 0.5536973137638624, + "2061635888/train0": 0.5639694431421344, + "2061635888/train1": 0.5581364424191588, + "2061635888/train2": 0.5794706736680671, + "211245842/train0": 0.5426955700305087, + "211245842/train1": 0.5351083386938548, + "211245842/train2": 0.534871348160612, + "278008524/train0": 0.5909019399202626, + "278008524/train1": 0.5847988892880861, + "278008524/train2": 0.59445093065676, + "307068879/train0": 0.5728850640530763, + "307068879/train1": 0.5704103700900957, + "307068879/train2": 0.5723883106472546, + "412102563/train0": 0.5493461173759878, + "412102563/train1": 0.5593981622164304, + "412102563/train2": 0.5532244830443317, + "440577711/train0": 0.5624630563402808, + "440577711/train1": 0.5471459072582541, + "440577711/train2": 0.5475473443233374, + "441046872/train0": 0.5664245191730715, + "441046872/train1": 0.5652066435973622, + "441046872/train2": 0.5755444444938004, + "509430392/train0": 0.5365694240669033, + "509430392/train1": 0.5326653211281538, + "509430392/train2": 0.5403636942457117, + "585904130/train0": 0.5472546704459165, + "585904130/train1": 0.549320518336562, + "585904130/train2": 0.5513841945828967, + "608016290/train0": 0.594216772727526, + "608016290/train1": 0.5978828235074669, + "608016290/train2": 0.6068596136210345, + "626486813/train0": 0.6158650684827733, + "626486813/train1": 0.6045161568297507, + "626486813/train2": 0.6142120016040145, + "636680137/train0": 0.540894445846163, + "636680137/train1": 0.5301042033378723, + "636680137/train2": 0.5238615170185813, + "655884605/train0": 0.5958635862373126, + "655884605/train1": 0.6117746577593474, + "655884605/train2": 0.5899703889806722, + "672328262/train0": 0.5940991028072462, + "672328262/train1": 0.5924504110795782, + "672328262/train2": 0.6012678269199632, + "701667486/train0": 0.5278746131288938, + "701667486/train1": 0.5308683756394741, + "701667486/train2": 0.5331361711412006, + "747260437/train0": 0.5322967641018285, + "747260437/train1": 0.5502411276617256, + "747260437/train2": 0.5279460992108118, + "750448296/train0": 0.5359906885657441, + "750448296/train1": 0.5377836086909143, + "750448296/train2": 0.5376337500179175, + "767747424/train0": 0.5592639661990686, + "767747424/train1": 0.5526263680260839, + "767747424/train2": 0.5607639712421639 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5621158691203029, + "low": 0.5492708813906398, + "mean": 0.5555302194109235, + "n": 5, + "num_samples": 300 + }, + "mean": 0.5555302194109235, + "per_task": { + "LiftPegUpright-v1": 0.5651094024103738, + "PickCube-v1": 0.5553133321393634, + "PullCube-v1": 0.5584521161129652, + "PushCube-v1": 0.5419772241344875, + "StackCube-v1": 0.5567990222574275 + } + }, + "metric": "pairwise_causal_calibration_confidence", + "micro": { + "confidence": 0.95, + "high": 0.5613581362340928, + "low": 0.5522925498793144, + "mean": 0.5565632957484361, + "n": 138, + "num_samples": 300 + }, + "n": 138 + }, + "pairwise_causal_calibration_ece": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.14460797017020918, + "low": 0.11646635724751547, + "mean": 0.13076108178779555, + "n": 120, + "num_samples": 300 + }, + "mean": 0.13076108178779555, + "per_seed": { + "1073161778/train0": 0.06721196591303755, + "1073161778/train1": 0.14187924624189804, + "1073161778/train2": 0.051910371125031565, + "1113371261/train0": 0.08992883305237898, + "1113371261/train1": 0.12516937728295682, + "1113371261/train2": 0.11947171326887146, + "1151410346/train0": 0.007632155302273683, + "1151410346/train1": 0.16877884785991246, + "1151410346/train2": 0.15903865402494677, + "1261930456/train0": 0.09695325258934998, + "1261930456/train1": 0.05726976772179637, + "1261930456/train2": 0.0620734765031595, + "1324133122/train0": 0.1479719277879018, + "1324133122/train1": 0.07659820557682445, + "1324133122/train2": 0.21571602442065527, + "1324642411/train0": 0.020540102462261362, + "1324642411/train1": 0.07839152778399854, + "1324642411/train2": 0.03459375413581838, + "1375492786/train0": 0.06738252647804421, + "1375492786/train1": 0.03869980516681758, + "1375492786/train2": 0.17084580857483078, + "141984615/train0": 0.2615138166625908, + "141984615/train1": 0.14136934515423885, + "141984615/train2": 0.1353553061579339, + "1461069713/train0": 0.14311635561767766, + "1461069713/train1": 0.19034912411756205, + "1461069713/train2": 0.019313884434419615, + "1480856810/train0": 0.2573289115592079, + "1480856810/train1": 0.04477881306501008, + "1480856810/train2": 0.03438565611403155, + "1509980664/train0": 0.10792743584436665, + "1509980664/train1": 0.36122831199394134, + "1509980664/train2": 0.024528598759442233, + "1522272686/train0": 0.11611644918527875, + "1522272686/train1": 0.13521676734451044, + "1522272686/train2": 0.06840618279050634, + "1549324805/train0": 0.10371428205750155, + "1549324805/train1": 0.2354300826197445, + "1549324805/train2": 0.08175730154924889, + "1578990937/train0": 0.15924329200874998, + "1578990937/train1": 0.15533501053451115, + "1578990937/train2": 0.13583474644587978, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.23970275130369073, + "1790188787/train1": 0.16632134275257193, + "1790188787/train2": 0.08428348319847451, + "1822210471/train0": 0.0719433679479887, + "1822210471/train1": 0.10100358053694718, + "1822210471/train2": 0.16770661828347844, + "1845489013/train0": 0.10301078523865043, + "1845489013/train1": 0.24397886450719253, + "1845489013/train2": 0.19255415849493018, + "1999022574/train0": 0.1860652464224708, + "1999022574/train1": 0.13139276293223526, + "1999022574/train2": 0.14831812450150075, + "2012211287/train0": 0.04620576008955871, + "2012211287/train1": 0.07041895983827043, + "2012211287/train2": 0.2313517331051612, + "2038594191/train0": 0.31209417490640445, + "2038594191/train1": 0.37103397324370413, + "2038594191/train2": 0.19145666287357221, + "2054899126/train0": 0.13126153408916094, + "2054899126/train1": 0.07456602027416159, + "2054899126/train2": 0.026703688149182295, + "2061635888/train0": 0.130957329325739, + "2061635888/train1": 0.1414697757524921, + "2061635888/train2": 0.13502622922362276, + "211245842/train0": 0.08755653080982741, + "211245842/train1": 0.12495817852344455, + "211245842/train2": 0.009871348160611883, + "278008524/train0": 0.1344967059497781, + "278008524/train1": 0.06891187442684295, + "278008524/train2": 0.09739801499095443, + "307068879/train0": 0.1199981805489268, + "307068879/train1": 0.07309450817078264, + "307068879/train2": 0.1767839150428592, + "412102563/train0": 0.10951912707521982, + "412102563/train1": 0.23547363265536436, + "412102563/train2": 0.0651292449490937, + "440577711/train0": 0.09270935745282255, + "440577711/train1": 0.3939678976832891, + "440577711/train2": 0.17413687717265497, + "441046872/train0": 0.19522652344582997, + "441046872/train1": 0.09055210003018804, + "441046872/train2": 0.13114432369738743, + "509430392/train0": 0.10561970831698192, + "509430392/train1": 0.038763250300417725, + "509430392/train2": 0.14212516552457483, + "585904130/train0": 0.17820705139829746, + "585904130/train1": 0.19650416069860938, + "585904130/train2": 0.1758885326898305, + "608016290/train0": 0.11412216411454103, + "608016290/train1": 0.20716363895507375, + "608016290/train2": 0.0651220182950748, + "626486813/train0": 0.1759930830571895, + "626486813/train1": 0.2753494901630838, + "626486813/train2": 0.14970168842483308, + "636680137/train0": 0.11589444584616293, + "636680137/train1": 0.09489579666212766, + "636680137/train2": 0.048861517018581324, + "655884605/train0": 0.10068633489228643, + "655884605/train1": 0.228982599451425, + "655884605/train2": 0.19094254082377798, + "672328262/train0": 0.22943030895745967, + "672328262/train1": 0.07756824536937998, + "672328262/train2": 0.4012678269199631, + "701667486/train0": 0.1469222321765129, + "701667486/train1": 0.004845910074811617, + "701667486/train2": 0.08869172669675614, + "747260437/train0": 0.046545342019779196, + "747260437/train1": 0.2890079292574854, + "747260437/train2": 0.1803872341225216, + "750448296/train0": 0.07322390346905903, + "750448296/train1": 0.09632988234770806, + "750448296/train2": 0.07076889063044275, + "767747424/train0": 0.1274080019549793, + "767747424/train1": 0.15613421435842795, + "767747424/train2": 0.10791269448114733 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.1292576743373383, + "low": 0.11373643350416622, + "mean": 0.12135669260871884, + "n": 5, + "num_samples": 300 + }, + "mean": 0.12135669260871884, + "per_task": { + "LiftPegUpright-v1": 0.11881236057424142, + "PickCube-v1": 0.10657979977343184, + "PullCube-v1": 0.13244853449538532, + "PushCube-v1": 0.11799694975778399, + "StackCube-v1": 0.13094581844275163 + } + }, + "metric": "pairwise_causal_calibration_ece", + "micro": { + "confidence": 0.95, + "high": 0.1433183357132694, + "low": 0.11511341593299673, + "mean": 0.12833834721763002, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "pairwise_causal_calibration_pairs": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 88.32847222222222, + "low": 75.17222222222223, + "mean": 82.21180555555556, + "n": 120, + "num_samples": 300 + }, + "mean": 82.21180555555556, + "per_seed": { + "1073161778/train0": 119.0, + "1073161778/train1": 120.0, + "1073161778/train2": 84.0, + "1113371261/train0": 28.0, + "1113371261/train1": 28.0, + "1113371261/train2": 28.0, + "1151410346/train0": 105.0, + "1151410346/train1": 105.0, + "1151410346/train2": 105.0, + "1261930456/train0": 99.0, + "1261930456/train1": 110.0, + "1261930456/train2": 84.0, + "1324133122/train0": 28.0, + "1324133122/train1": 28.0, + "1324133122/train2": 28.0, + "1324642411/train0": 120.0, + "1324642411/train1": 119.0, + "1324642411/train2": 110.0, + "1375492786/train0": 75.0, + "1375492786/train1": 114.0, + "1375492786/train2": 84.0, + "141984615/train0": 120.0, + "141984615/train1": 120.0, + "141984615/train2": 120.0, + "1461069713/train0": 120.0, + "1461069713/train1": 120.0, + "1461069713/train2": 120.0, + "1480856810/train0": 99.0, + "1480856810/train1": 119.0, + "1480856810/train2": 119.0, + "1509980664/train0": 90.0, + "1509980664/train1": 63.0, + "1509980664/train2": 90.0, + "1522272686/train0": 36.0, + "1522272686/train1": 36.0, + "1522272686/train2": 36.0, + "1549324805/train0": 120.0, + "1549324805/train1": 120.0, + "1549324805/train2": 120.0, + "1578990937/train0": 117.0, + "1578990937/train1": 110.0, + "1578990937/train2": 119.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 110.0, + "1790188787/train1": 114.0, + "1790188787/train2": 110.0, + "1822210471/train0": 40.0, + "1822210471/train1": 49.0, + "1822210471/train2": 40.0, + "1845489013/train0": 46.0, + "1845489013/train1": 13.0, + "1845489013/train2": 25.0, + "1999022574/train0": 70.0, + "1999022574/train1": 70.0, + "1999022574/train2": 76.0, + "2012211287/train0": 119.0, + "2012211287/train1": 120.0, + "2012211287/train2": 92.0, + "2038594191/train0": 29.0, + "2038594191/train1": 54.0, + "2038594191/train2": 99.0, + "2054899126/train0": 60.0, + "2054899126/train1": 57.0, + "2054899126/train2": 59.5, + "2061635888/train0": 36.0, + "2061635888/train1": 36.0, + "2061635888/train2": 36.0, + "211245842/train0": 119.0, + "211245842/train1": 120.0, + "211245842/train2": 120.0, + "278008524/train0": 109.5, + "278008524/train1": 106.0, + "278008524/train2": 109.5, + "307068879/train0": 91.0, + "307068879/train1": 91.0, + "307068879/train2": 91.0, + "412102563/train0": 60.0, + "412102563/train1": 39.0, + "412102563/train2": 84.0, + "440577711/train0": 29.0, + "440577711/train1": 15.0, + "440577711/train2": 54.0, + "441046872/train0": 55.0, + "441046872/train1": 55.0, + "441046872/train2": 55.0, + "509430392/train0": 119.0, + "509430392/train1": 105.0, + "509430392/train2": 119.0, + "585904130/train0": 84.0, + "585904130/train1": 75.0, + "585904130/train2": 110.0, + "608016290/train0": 105.0, + "608016290/train1": 105.0, + "608016290/train2": 120.0, + "626486813/train0": 120.0, + "626486813/train1": 120.0, + "626486813/train2": 120.0, + "636680137/train0": 120.0, + "636680137/train1": 120.0, + "636680137/train2": 120.0, + "655884605/train0": 36.0, + "655884605/train1": 36.0, + "655884605/train2": 36.0, + "672328262/train0": 17.0, + "672328262/train1": 35.0, + "672328262/train2": 30.0, + "701667486/train0": 42.0, + "701667486/train1": 84.0, + "701667486/train2": 54.0, + "747260437/train0": 120.0, + "747260437/train1": 120.0, + "747260437/train2": 120.0, + "750448296/train0": 108.0, + "750448296/train1": 101.66666666666667, + "750448296/train2": 108.0, + "767747424/train0": 119.25, + "767747424/train1": 119.0, + "767747424/train2": 119.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 95.32888888888888, + "low": 54.05090909090909, + "mean": 77.60379797979797, + "n": 5, + "num_samples": 300 + }, + "mean": 77.60379797979797, + "per_task": { + "LiftPegUpright-v1": 94.8, + "PickCube-v1": 36.0, + "PullCube-v1": 92.32, + "PushCube-v1": 97.44444444444444, + "StackCube-v1": 67.45454545454545 + } + }, + "metric": "pairwise_causal_calibration_pairs", + "micro": { + "confidence": 0.95, + "high": 92.36111111111111, + "low": 80.19444444444444, + "mean": 86.34722222222223, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9056800281312739, + "low": 0.7479934311770733, + "mean": 0.8247400227037968, + "n": 120, + "num_samples": 300 + }, + "mean": 0.8247400227037968, + "per_seed": { + "1073161778/train0": 0.8315109279797959, + "1073161778/train1": 0.696723804413939, + "1073161778/train2": 0.8010166202874374, + "1113371261/train0": 0.028473507842246004, + "1113371261/train1": 0.018305243090398104, + "1113371261/train2": -0.16653094820357445, + "1151410346/train0": 0.8265443858364611, + "1151410346/train1": 0.9117051526313378, + "1151410346/train2": 0.8848141434142904, + "1261930456/train0": 0.7394668700396011, + "1261930456/train1": 0.7193134962639322, + "1261930456/train2": 0.7862881124399304, + "1324133122/train0": -0.3146091278603853, + "1324133122/train1": 0.05209384637709236, + "1324133122/train2": -0.31434308061981353, + "1324642411/train0": 1.8003040878491947, + "1324642411/train1": 1.3876858687209335, + "1324642411/train2": 0.7578621304129007, + "1375492786/train0": 0.13235098862844796, + "1375492786/train1": -0.48662334960024145, + "1375492786/train2": -0.36262329065526533, + "141984615/train0": 0.5415586302800286, + "141984615/train1": 0.8524230828537844, + "141984615/train2": 0.3015705513281932, + "1461069713/train0": -0.06695685840699794, + "1461069713/train1": 0.24721792540099097, + "1461069713/train2": 0.2696456255951446, + "1480856810/train0": 2.2046095259547753, + "1480856810/train1": 1.735589926228614, + "1480856810/train2": 1.359251321625822, + "1509980664/train0": 0.6801689818523341, + "1509980664/train1": 0.7807195223473116, + "1509980664/train2": 0.7965612115773528, + "1522272686/train0": 0.3705665707855397, + "1522272686/train1": 0.8136424781392986, + "1522272686/train2": 0.5686852402790281, + "1549324805/train0": 1.332576441483642, + "1549324805/train1": 0.9503772340326878, + "1549324805/train2": 1.4498268017490537, + "1578990937/train0": 1.3204376505810713, + "1578990937/train1": 1.2385033751871737, + "1578990937/train2": 1.1890371636445307, + "1747178030/train0": 0.6807136864797709, + "1747178030/train1": 0.7412627528047919, + "1747178030/train2": 0.7399153467725452, + "1790188787/train0": 1.1652542261433299, + "1790188787/train1": 1.1907212928313995, + "1790188787/train2": 1.0616754451066321, + "1822210471/train0": 0.7418095681624295, + "1822210471/train1": 0.7078283118607503, + "1822210471/train2": 0.714483922998961, + "1845489013/train0": 0.798908624848506, + "1845489013/train1": 0.8663205932771525, + "1845489013/train2": 0.8229653113797162, + "1999022574/train0": 0.8291022763943762, + "1999022574/train1": 0.7563954831040236, + "1999022574/train2": 0.8419564487563823, + "2012211287/train0": 1.107133532936841, + "2012211287/train1": 0.9306280702125403, + "2012211287/train2": 1.1151401969224932, + "2038594191/train0": 1.2008765491740636, + "2038594191/train1": 1.112157225492381, + "2038594191/train2": 1.2364955407573137, + "2054899126/train0": 1.014453498377106, + "2054899126/train1": 1.074027910955616, + "2054899126/train2": 1.0710380363686038, + "2061635888/train0": 0.39584763737906137, + "2061635888/train1": 0.3705725365164993, + "2061635888/train2": 0.345599063353308, + "211245842/train0": 1.1272780451605109, + "211245842/train1": 1.2265101367583866, + "211245842/train2": 1.1256343724003985, + "278008524/train0": 0.27572048526322235, + "278008524/train1": 0.3598249612248885, + "278008524/train2": 0.4752736549448759, + "307068879/train0": 0.7747416665744656, + "307068879/train1": 0.3600826916101192, + "307068879/train2": 0.979654396381198, + "412102563/train0": 1.0435143603129777, + "412102563/train1": 0.949568847497014, + "412102563/train2": 0.9787258921638226, + "440577711/train0": 1.7706684649919293, + "440577711/train1": 1.808986225138223, + "440577711/train2": 1.7310319203842686, + "441046872/train0": 0.6069194020559678, + "441046872/train1": 0.1629836409186699, + "441046872/train2": 0.5214657402479244, + "509430392/train0": 1.7250273458520244, + "509430392/train1": 1.5703557578982594, + "509430392/train2": 1.3794654730008236, + "585904130/train0": 0.9073249211388359, + "585904130/train1": 0.8486373501760932, + "585904130/train2": 0.9433176675944015, + "608016290/train0": 1.589765155203597, + "608016290/train1": 1.3392733370703795, + "608016290/train2": 1.4772393565445312, + "626486813/train0": 0.9601260167395063, + "626486813/train1": 0.9603159694517454, + "626486813/train2": 0.7735840442684991, + "636680137/train0": 1.0758545824030454, + "636680137/train1": 0.44432064636845636, + "636680137/train2": 0.5548863156073045, + "655884605/train0": 0.5292851601669687, + "655884605/train1": 0.8854182759065323, + "655884605/train2": 0.7967317370820184, + "672328262/train0": 0.7871393331842744, + "672328262/train1": 0.7852289914727505, + "672328262/train2": 0.6365332388231544, + "701667486/train0": 1.1356123561511042, + "701667486/train1": 1.234980143424118, + "701667486/train2": 1.3518684532465013, + "747260437/train0": 0.7403769985406177, + "747260437/train1": 0.6210780034951271, + "747260437/train2": 0.9133990308397049, + "750448296/train0": 0.8003581361777009, + "750448296/train1": 0.4055514323654445, + "750448296/train2": 0.5039790420368024, + "767747424/train0": 0.5457716259676435, + "767747424/train1": 0.6028818781511358, + "767747424/train2": 0.5655051629030958 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.1422375180911708, + "low": 0.6446955243124963, + "mean": 0.8504917118375032, + "n": 5, + "num_samples": 300 + }, + "mean": 0.8504917118375032, + "per_task": { + "LiftPegUpright-v1": 0.8862852359573267, + "PickCube-v1": 0.5842980964012888, + "PullCube-v1": 0.6759761060180944, + "PushCube-v1": 1.3829754161601442, + "StackCube-v1": 0.7229237046506615 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.8833200719102705, + "low": 0.7199132535269486, + "mean": 0.795107086974134, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.673611111111111, + "low": 0.4958333333333333, + "mean": 0.5847222222222223, + "n": 120, + "num_samples": 300 + }, + "mean": 0.5847222222222223, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7563636363636363, + "low": 0.22303030303030305, + "mean": 0.5074747474747474, + "n": 5, + "num_samples": 300 + }, + "mean": 0.5074747474747474, + "per_task": { + "LiftPegUpright-v1": 0.6, + "PickCube-v1": 0.0, + "PullCube-v1": 0.5333333333333333, + "PushCube-v1": 0.8888888888888888, + "StackCube-v1": 0.5151515151515151 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.6458333333333334, + "low": 0.4861111111111111, + "mean": 0.5694444444444444, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.4087191828565362, + "low": 1.0967581196351806, + "mean": 1.251578581963097, + "n": 120, + "num_samples": 300 + }, + "mean": 1.251578581963097, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.10274861752986908, + "1113371261/train1": 0.18093222379684448, + "1113371261/train2": 0.19527167081832886, + "1151410346/train0": 0.8663521409034729, + "1151410346/train1": 2.0, + "1151410346/train2": 0.5522047281265259, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.054989203810691833, + "1324133122/train1": 0.1324586570262909, + "1324133122/train2": 0.19291165471076965, + "1324642411/train0": 0.14406004548072815, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.015177905559539795, + "141984615/train1": 0.037011127918958664, + "141984615/train2": 0.058951716870069504, + "1461069713/train0": 0.06019836664199829, + "1461069713/train1": 0.016585012897849083, + "1461069713/train2": 0.09997648000717163, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 0.18325337767601013, + "1522272686/train1": 0.07014383375644684, + "1522272686/train2": 0.14937306940555573, + "1549324805/train0": 0.060871422290802, + "1549324805/train1": 0.19892151653766632, + "1549324805/train2": 0.11310184001922607, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.077283151447773, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.09799930453300476, + "2061635888/train1": 0.07580336928367615, + "2061635888/train2": 0.11105132102966309, + "211245842/train0": 2.0, + "211245842/train1": 0.21295717358589172, + "211245842/train2": 0.23388276994228363, + "278008524/train0": 1.0733563229441643, + "278008524/train1": 1.0376745946705341, + "278008524/train2": 1.1359489113092422, + "307068879/train0": 0.2115575522184372, + "307068879/train1": 0.5876073241233826, + "307068879/train2": 0.2302134484052658, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 0.7233588695526123, + "441046872/train2": 0.7191348671913147, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.7117276191711426, + "626486813/train0": 0.04469547560438514, + "626486813/train1": 0.035827874671667814, + "626486813/train2": 0.04489925503730774, + "636680137/train0": 0.08826130628585815, + "636680137/train1": 0.06741118431091309, + "636680137/train2": 0.0756516233086586, + "655884605/train0": 0.13732868432998657, + "655884605/train1": 0.8128758072853088, + "655884605/train2": 0.21447330713272095, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.10839563608169556, + "747260437/train1": 0.06328952312469482, + "747260437/train2": 0.12510187923908234, + "750448296/train0": 0.7526603279014429, + "750448296/train1": 1.3705032741030057, + "750448296/train2": 1.3504486680030823, + "767747424/train0": 1.0424291878007352, + "767747424/train1": 1.0629530912265182, + "767747424/train2": 1.0671670949086547 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.5757816767742236, + "low": 0.5943610860542816, + "mean": 1.1196794969856438, + "n": 5, + "num_samples": 300 + }, + "mean": 1.1196794969856438, + "per_task": { + "LiftPegUpright-v1": 1.4122399787108104, + "PickCube-v1": 0.13425676027933756, + "PullCube-v1": 1.1093241602430741, + "PushCube-v1": 1.7857814149724112, + "StackCube-v1": 1.1567951707225856 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.3550500385204538, + "low": 1.0635484514318199, + "mean": 1.2159999172436073, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 6.045833333333333, + "low": 4.645833333333333, + "mean": 5.317361111111111, + "n": 120, + "num_samples": 300 + }, + "mean": 5.317361111111111, + "per_seed": { + "1073161778/train0": 6.0, + "1073161778/train1": 8.0, + "1073161778/train2": 3.0, + "1113371261/train0": 2.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 12.0, + "1151410346/train1": 4.0, + "1151410346/train2": 6.0, + "1261930456/train0": 1.0, + "1261930456/train1": 2.0, + "1261930456/train2": 4.0, + "1324133122/train0": 6.0, + "1324133122/train1": 2.0, + "1324133122/train2": 1.0, + "1324642411/train0": 11.0, + "1324642411/train1": 4.0, + "1324642411/train2": 2.0, + "1375492786/train0": 3.0, + "1375492786/train1": 15.0, + "1375492786/train2": 5.0, + "141984615/train0": 3.0, + "141984615/train1": 5.0, + "141984615/train2": 10.0, + "1461069713/train0": 7.0, + "1461069713/train1": 15.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 10.0, + "1480856810/train2": 4.0, + "1509980664/train0": 2.0, + "1509980664/train1": 9.0, + "1509980664/train2": 9.0, + "1522272686/train0": 8.0, + "1522272686/train1": 7.0, + "1522272686/train2": 7.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 9.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 3.0, + "1747178030/train1": 3.0, + "1747178030/train2": 15.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 3.0, + "1822210471/train0": 1.0, + "1822210471/train1": 2.0, + "1822210471/train2": 7.0, + "1845489013/train0": 5.0, + "1845489013/train1": 7.0, + "1845489013/train2": 8.0, + "1999022574/train0": 3.0, + "1999022574/train1": 5.0, + "1999022574/train2": 13.0, + "2012211287/train0": 7.0, + "2012211287/train1": 4.0, + "2012211287/train2": 10.0, + "2038594191/train0": 1.0, + "2038594191/train1": 9.0, + "2038594191/train2": 2.0, + "2054899126/train0": 4.0, + "2054899126/train1": 11.0, + "2054899126/train2": 2.0, + "2061635888/train0": 1.0, + "2061635888/train1": 2.0, + "2061635888/train2": 7.0, + "211245842/train0": 5.0, + "211245842/train1": 1.0, + "211245842/train2": 7.0, + "278008524/train0": 10.0, + "278008524/train1": 4.5, + "278008524/train2": 14.0, + "307068879/train0": 4.0, + "307068879/train1": 9.0, + "307068879/train2": 10.0, + "412102563/train0": 10.0, + "412102563/train1": 8.0, + "412102563/train2": 9.0, + "440577711/train0": 1.0, + "440577711/train1": 4.0, + "440577711/train2": 2.0, + "441046872/train0": 7.0, + "441046872/train1": 6.0, + "441046872/train2": 1.0, + "509430392/train0": 15.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 4.0, + "585904130/train1": 15.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 3.0, + "608016290/train2": 1.0, + "626486813/train0": 9.5, + "626486813/train1": 6.0, + "626486813/train2": 2.0, + "636680137/train0": 0.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 0.0, + "655884605/train1": 6.0, + "655884605/train2": 1.0, + "672328262/train0": 7.0, + "672328262/train1": 4.0, + "672328262/train2": 4.0, + "701667486/train0": 10.0, + "701667486/train1": 11.0, + "701667486/train2": 15.0, + "747260437/train0": 9.0, + "747260437/train1": 4.0, + "747260437/train2": 14.0, + "750448296/train0": 0.0, + "750448296/train1": 7.333333333333333, + "750448296/train2": 9.0, + "767747424/train0": 8.25, + "767747424/train1": 3.75, + "767747424/train2": 8.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 6.545454545454545, + "low": 4.497333333333333, + "mean": 5.55389898989899, + "n": 5, + "num_samples": 300 + }, + "mean": 5.55389898989899, + "per_task": { + "LiftPegUpright-v1": 5.333333333333333, + "PickCube-v1": 7.333333333333333, + "PullCube-v1": 5.986666666666666, + "PushCube-v1": 3.7222222222222223, + "StackCube-v1": 5.393939393939394 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 6.263888888888889, + "low": 4.784722222222222, + "mean": 5.527777777777778, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.44166666666666665, + "low": 0.26666666666666666, + "mean": 0.35, + "n": 120, + "num_samples": 300 + }, + "mean": 0.35, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 1.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 0.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4545454545454545, + "low": 0.12121212121212119, + "mean": 0.290989898989899, + "n": 5, + "num_samples": 300 + }, + "mean": 0.290989898989899, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.29333333333333333, + "PushCube-v1": 0.5555555555555556, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.3819444444444444, + "low": 0.24305555555555555, + "mean": 0.3194444444444444, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.10625, + "low": -0.025, + "mean": 0.04375, + "n": 120, + "num_samples": 300 + }, + "mean": 0.04375, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 1.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": -1.0, + "2038594191/train2": 0.0, + "2054899126/train0": -0.5, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": -1.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": -1.0, + "747260437/train0": -1.0, + "747260437/train1": -1.0, + "747260437/train2": -1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": -0.25, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.26, + "low": -0.026666666666666665, + "mean": 0.09111111111111111, + "n": 5, + "num_samples": 300 + }, + "mean": 0.09111111111111111, + "per_task": { + "LiftPegUpright-v1": 0.13333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": -0.06666666666666667, + "PushCube-v1": 0.3888888888888889, + "StackCube-v1": 0.0 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.09027777777777778, + "low": -0.027777777777777776, + "mean": 0.027777777777777776, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.0168445986038326, + "low": 0.6897832043572433, + "mean": 0.8491037837639548, + "n": 120, + "num_samples": 300 + }, + "mean": 0.8491037837639548, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.12164360284805298, + "1324642411/train1": 0.11658626794815063, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.027254482731223106, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 0.21894000470638275, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.0260902252048254, + "1549324805/train1": 0.0303483996540308, + "1549324805/train2": 0.04690708592534065, + "1578990937/train0": 2.0, + "1578990937/train1": 0.12732218205928802, + "1578990937/train2": 0.13359522819519043, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 0.10585354268550873, + "1790188787/train2": 0.1341685652732849, + "1822210471/train0": 0.06306678056716919, + "1822210471/train1": 2.0, + "1822210471/train2": 0.0730673298239708, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.713559627532959, + "2012211287/train1": 0.7183246612548828, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 0.07674666494131088, + "2038594191/train2": 2.0, + "2054899126/train0": 1.0643903091549873, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.8592818975448608, + "211245842/train1": 0.09285585582256317, + "211245842/train2": 0.13431161642074585, + "278008524/train0": 0.05350120924413204, + "278008524/train1": 0.034389604814350605, + "278008524/train2": 0.047564467415213585, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.2021283358335495, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 0.11310504376888275, + "509430392/train1": 2.0, + "509430392/train2": 2.682209014892578e-07, + "585904130/train0": 0.09919407218694687, + "585904130/train1": 0.04792279005050659, + "585904130/train2": 2.0, + "608016290/train0": 0.707331120967865, + "608016290/train1": 2.0, + "608016290/train2": 0.6789124011993408, + "626486813/train0": 0.007258951431140304, + "626486813/train1": 0.011037707328796387, + "626486813/train2": 0.012973218690603971, + "636680137/train0": 0.056092143058776855, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 0.04013325646519661, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 0.8125, + "747260437/train0": 0.07023122161626816, + "747260437/train1": 0.05181918665766716, + "747260437/train2": 0.0880771279335022, + "750448296/train0": 0.09101381525397301, + "750448296/train1": 0.10720233743389447, + "750448296/train2": 0.09631064782540004, + "767747424/train0": 0.0617859915655572, + "767747424/train1": 0.5621845133136958, + "767747424/train2": 0.567537541501224 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0499388170904582, + "low": 0.6472050376432016, + "mean": 0.8435512482026176, + "n": 5, + "num_samples": 300 + }, + "mean": 0.8435512482026176, + "per_task": { + "LiftPegUpright-v1": 1.088020184636116, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.6341658942727372, + "PushCube-v1": 1.162042656706439, + "StackCube-v1": 0.7495684027671814 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.9005737514243164, + "low": 0.6412263065835254, + "mean": 0.7728274115272344, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.16965195490823437, + "low": -0.06579780244865105, + "mean": 0.058985721466160615, + "n": 120, + "num_samples": 300 + }, + "mean": 0.058985721466160615, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.015173077583312988, + "1324642411/train1": 0.010115742683410645, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": -0.07557976432144642, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.9129659235477448, + "1480856810/train1": 1.9129659235477448, + "1480856810/train2": 0.1319059282541275, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": -0.07842461206018925, + "1549324805/train1": -0.07416643761098385, + "1549324805/train2": -0.057607751339673996, + "1578990937/train0": 1.17586350440979, + "1578990937/train1": -0.6968143135309219, + "1578990937/train2": -0.6905412673950195, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.87523253262043, + "1790188787/train1": -0.01891392469406128, + "1790188787/train2": 0.009401097893714905, + "1822210471/train0": -0.008848525583744049, + "1822210471/train1": 1.9280846938490868, + "1822210471/train2": 0.0011520236730575562, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": -0.01004570722579956, + "2012211287/train1": -0.005280673503875732, + "2012211287/train2": 1.2763946652412415, + "2038594191/train0": 0.0, + "2038594191/train1": -1.9232533350586891, + "2038594191/train2": 0.0, + "2054899126/train0": -0.9356096908450127, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.6683734804391861, + "211245842/train1": -0.09805256128311157, + "211245842/train2": -0.056596800684928894, + "278008524/train0": 0.010766219347715378, + "278008524/train1": -0.008345385082066059, + "278008524/train2": 0.004829477518796921, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": -0.39796222746372223, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.9638472199440002, + "440577711/train1": 1.9638472199440002, + "440577711/train2": 1.9638472199440002, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.11310482025146484, + "509430392/train1": 1.999999776482582, + "509430392/train2": 4.470348358154297e-08, + "585904130/train0": -0.0825214758515358, + "585904130/train1": -0.13379275798797607, + "585904130/train2": 1.8182844519615173, + "608016290/train0": -0.018370509147644043, + "608016290/train1": 1.274298369884491, + "608016290/train2": -0.04678922891616821, + "626486813/train0": 0.002160608710255474, + "626486813/train1": 0.005939364607911557, + "626486813/train2": 0.007874875969719142, + "636680137/train0": -0.027042903006076813, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": -1.9598667435348034, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": -1.1875, + "747260437/train0": -1.9297687783837318, + "747260437/train1": -1.9481808133423328, + "747260437/train2": -1.9119228720664978, + "750448296/train0": -0.026579334090153377, + "750448296/train1": -0.010390811910231909, + "750448296/train2": -0.02128250151872635, + "767747424/train0": -0.5114133504393976, + "767747424/train1": -0.011014828691259027, + "767747424/train2": -0.005661800503730774 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4840477555899909, + "low": -0.06713234748386523, + "mean": 0.14799145102860276, + "n": 5, + "num_samples": 300 + }, + "mean": 0.14799145102860276, + "per_task": { + "LiftPegUpright-v1": 0.1381496459245682, + "PickCube-v1": 0.0, + "PullCube-v1": -0.14100456695072353, + "PushCube-v1": 0.7696384779281087, + "StackCube-v1": -0.02682630175893957 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.13739498196405797, + "low": -0.08578009277456052, + "mean": 0.031007825084897276, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5576142528169536, + "low": 0.31175716476257853, + "mean": 0.4202619706426857, + "n": 120, + "num_samples": 300 + }, + "mean": 0.4202619706426857, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.02250880002975464, + "1113371261/train2": 0.036848247051239014, + "1151410346/train0": 0.6403508484363556, + "1151410346/train1": 1.7739987075328827, + "1151410346/train2": 0.32620343565940857, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.04025374352931976, + "1324133122/train2": 0.10070674121379852, + "1324642411/train0": 0.02241644263267517, + "1324642411/train1": 1.8834137320518494, + "1324642411/train2": 1.89352947473526, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.009756645187735558, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.006073415279388428, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.7810599952936172, + "1509980664/train0": 1.1665899753570557, + "1509980664/train1": 1.1665899753570557, + "1509980664/train2": 1.1665899753570557, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0347811970859766, + "1549324805/train1": 0.16857311688363552, + "1549324805/train2": 0.06619475409388542, + "1578990937/train0": 0.0, + "1578990937/train1": 1.872677817940712, + "1578990937/train2": 1.8664047718048096, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.8941464573144913, + "1790188787/train2": 1.865831434726715, + "1822210471/train0": 1.9369332194328308, + "1822210471/train1": 0.0, + "1822210471/train2": 1.9269326701760292, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.286440372467041, + "2012211287/train1": 1.2816753387451172, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 1.9232533350586891, + "2038594191/train2": 0.0, + "2054899126/train0": 0.012892842292785645, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.04574880003929138, + "2061635888/train1": 0.02355286478996277, + "2061635888/train2": 0.05880081653594971, + "211245842/train0": 1.1407181024551392, + "211245842/train1": 0.12010131776332855, + "211245842/train2": 0.09957115352153778, + "278008524/train0": 1.0198551137000322, + "278008524/train1": 1.0032849898561835, + "278008524/train2": 1.0883844438940287, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.02808511257171631, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.3000448346138, + "441046872/train1": 0.023403704166412354, + "441046872/train2": 0.019179701805114746, + "509430392/train0": 1.8868949562311172, + "509430392/train1": 0.0, + "509430392/train2": 5.155801773071289e-06, + "585904130/train0": 1.9008059278130531, + "585904130/train1": 1.9520772099494934, + "585904130/train2": 0.0, + "608016290/train0": 1.292668879032135, + "608016290/train1": 0.0, + "608016290/train2": 0.03281521797180176, + "626486813/train0": 0.037436524173244834, + "626486813/train1": 0.024790167342871428, + "626486813/train2": 0.03192603634670377, + "636680137/train0": 0.0321691632270813, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.6498691290616989, + "655884605/train2": 0.05146662890911102, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 1.9598667435348034, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 1.1875, + "747260437/train0": 0.0381644144654274, + "747260437/train1": 0.011470336467027664, + "747260437/train2": 0.03702475130558014, + "750448296/train0": 0.6616465126474699, + "750448296/train1": 1.2633009366691113, + "750448296/train2": 1.2541380201776822, + "767747424/train0": 0.980643196235178, + "767747424/train1": 0.5007685779128224, + "767747424/train2": 0.49962955340743065 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5367166141374037, + "low": 0.1779043208972805, + "mean": 0.3728429147021167, + "n": 5, + "num_samples": 300 + }, + "mean": 0.3728429147021167, + "per_task": { + "LiftPegUpright-v1": 0.3509542107582092, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4787018513850247, + "PushCube-v1": 0.6237387582659721, + "StackCube-v1": 0.41081975310137775 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.587600458064925, + "low": 0.3392519335482373, + "mean": 0.45799514941932606, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3125, + "low": 0.16597222222222227, + "mean": 0.23472222222222222, + "n": 120, + "num_samples": 300 + }, + "mean": 0.23472222222222222, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 0.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 1.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 0.0, + "608016290/train0": 1.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 1.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3013333333333333, + "low": 0.10181818181818181, + "mean": 0.21648484848484845, + "n": 5, + "num_samples": 300 + }, + "mean": 0.21648484848484845, + "per_task": { + "LiftPegUpright-v1": 0.26666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.24, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.24242424242424243 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.3194444444444444, + "low": 0.1875, + "mean": 0.25, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3055555555555555, + "low": 0.15833333333333333, + "mean": 0.23194444444444443, + "n": 120, + "num_samples": 300 + }, + "mean": 0.23194444444444443, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 0.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7023030303030303, + "low": 0.10133333333333334, + "mean": 0.40452525252525257, + "n": 5, + "num_samples": 300 + }, + "mean": 0.40452525252525257, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 1.0, + "PullCube-v1": 0.02666666666666667, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": 0.48484848484848486 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.2708333333333333, + "low": 0.14583333333333334, + "mean": 0.2013888888888889, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5403802280805798, + "low": 0.2915130193099483, + "mean": 0.4232947583840643, + "n": 120, + "num_samples": 300 + }, + "mean": 0.4232947583840643, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.897251382470131, + "1113371261/train1": 1.8190677762031555, + "1113371261/train2": 1.8047283291816711, + "1151410346/train0": 1.133647859096527, + "1151410346/train1": 0.0, + "1151410346/train2": 1.4477952718734741, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.9450107961893082, + "1324133122/train1": 1.867541342973709, + "1324133122/train2": 1.8070883452892303, + "1324642411/train0": 1.8559399545192719, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.270599365234375, + "141984615/train1": 0.24876614287495613, + "141984615/train2": 0.2268255539238453, + "1461069713/train0": 0.0991440862417221, + "1461069713/train1": 0.14275743998587132, + "1461069713/train2": 0.05936597287654877, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.8167466223239899, + "1522272686/train1": 1.9298561662435532, + "1522272686/train2": 1.8506269305944443, + "1549324805/train0": 0.23658466339111328, + "1549324805/train1": 0.09853456914424896, + "1549324805/train2": 0.1843542456626892, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.922716848552227, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.9020006954669952, + "2061635888/train1": 1.9241966307163239, + "2061635888/train2": 1.888948678970337, + "211245842/train0": 0.0, + "211245842/train1": 1.7870428264141083, + "211245842/train2": 1.7661172300577164, + "278008524/train0": 0.04055231809616089, + "278008524/train1": 0.07623404636979103, + "278008524/train2": 0.0, + "307068879/train0": 1.7884424477815628, + "307068879/train1": 1.4123926758766174, + "307068879/train2": 1.7697865515947342, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.2766411304473877, + "441046872/train2": 1.2808651328086853, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.9999945759773254, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.2882723808288574, + "626486813/train0": 0.12161866808310151, + "626486813/train1": 0.12975450651720166, + "626486813/train2": 0.12068312615156174, + "636680137/train0": 0.0823030173778534, + "636680137/train1": 0.10315313935279846, + "636680137/train2": 0.09491270035505295, + "655884605/train0": 1.8626713156700134, + "655884605/train1": 1.1871241927146912, + "655884605/train2": 1.785526692867279, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.17026761174201965, + "747260437/train1": 0.21537372469902039, + "747260437/train2": 0.15356136858463287, + "750448296/train0": 0.6553361676633358, + "750448296/train1": 0.03749322146177292, + "750448296/train2": 0.05754782756169637, + "767747424/train0": 0.06562284985557199, + "767747424/train1": 0.045098946429789066, + "767747424/train2": 0.04088494274765253 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.3095993929816916, + "low": 0.20360193743308388, + "mean": 0.7236978183567826, + "n": 5, + "num_samples": 300 + }, + "mean": 0.7236978183567826, + "per_task": { + "LiftPegUpright-v1": 0.5877600212891897, + "PickCube-v1": 1.8657432397206624, + "PullCube-v1": 0.1075624164690574, + "PushCube-v1": 0.21421858502758873, + "StackCube-v1": 0.8432048292774142 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 0.49125471889015493, + "low": 0.27718254707158646, + "mean": 0.3761285081272945, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8498021009856326, + "low": 0.7752087490002378, + "mean": 0.8107982548634141, + "n": 120, + "num_samples": 300 + }, + "mean": 0.8107982548634141, + "per_seed": { + "1073161778/train0": 0.87335466614541, + "1073161778/train1": 0.87335466614541, + "1073161778/train2": 0.87335466614541, + "1113371261/train0": 1.0718654827809488, + "1113371261/train1": 1.0718654827809488, + "1113371261/train2": 1.0718654827809488, + "1151410346/train0": 1.0718654827809488, + "1151410346/train1": 1.0718654827809488, + "1151410346/train2": 1.0718654827809488, + "1261930456/train0": 0.8468573198810986, + "1261930456/train1": 0.8468573198810986, + "1261930456/train2": 0.8468573198810986, + "1324133122/train0": 1.0718654827809488, + "1324133122/train1": 1.0718654827809488, + "1324133122/train2": 1.0718654827809488, + "1324642411/train0": 0.8468573198810986, + "1324642411/train1": 0.8468573198810986, + "1324642411/train2": 0.8468573198810986, + "1375492786/train0": 0.5825884328057473, + "1375492786/train1": 0.5825884328057473, + "1375492786/train2": 0.5825884328057473, + "141984615/train0": 0.5825884328057473, + "141984615/train1": 0.5825884328057473, + "141984615/train2": 0.5825884328057473, + "1461069713/train0": 0.5825884328057473, + "1461069713/train1": 0.5825884328057473, + "1461069713/train2": 0.5825884328057473, + "1480856810/train0": 0.8468573198810986, + "1480856810/train1": 0.8468573198810986, + "1480856810/train2": 0.8468573198810986, + "1509980664/train0": 1.0718654827809488, + "1509980664/train1": 1.0718654827809488, + "1509980664/train2": 1.0718654827809488, + "1522272686/train0": 1.2894892762347252, + "1522272686/train1": 1.2894892762347252, + "1522272686/train2": 1.2894892762347252, + "1549324805/train0": 0.5825884328057473, + "1549324805/train1": 0.5825884328057473, + "1549324805/train2": 0.5825884328057473, + "1578990937/train0": 1.0718654827809488, + "1578990937/train1": 1.0718654827809488, + "1578990937/train2": 1.0718654827809488, + "1747178030/train0": 0.5825884328057473, + "1747178030/train1": 0.5825884328057473, + "1747178030/train2": 0.5825884328057473, + "1790188787/train0": 0.8468573198810986, + "1790188787/train1": 0.8468573198810986, + "1790188787/train2": 0.8468573198810986, + "1822210471/train0": 0.5825884328057473, + "1822210471/train1": 0.5825884328057473, + "1822210471/train2": 0.5825884328057473, + "1845489013/train0": 0.5825884328057473, + "1845489013/train1": 0.5825884328057473, + "1845489013/train2": 0.5825884328057473, + "1999022574/train0": 1.0718654827809488, + "1999022574/train1": 1.0718654827809488, + "1999022574/train2": 1.0718654827809488, + "2012211287/train0": 0.87335466614541, + "2012211287/train1": 0.87335466614541, + "2012211287/train2": 0.87335466614541, + "2038594191/train0": 0.5825884328057473, + "2038594191/train1": 0.5825884328057473, + "2038594191/train2": 0.5825884328057473, + "2054899126/train0": 0.5825884328057473, + "2054899126/train1": 0.5825884328057473, + "2054899126/train2": 0.5825884328057473, + "2061635888/train0": 1.0718654827809488, + "2061635888/train1": 1.0718654827809488, + "2061635888/train2": 1.0718654827809488, + "211245842/train0": 1.0718654827809488, + "211245842/train1": 1.0718654827809488, + "211245842/train2": 1.0718654827809488, + "278008524/train0": 0.5825884328057473, + "278008524/train1": 0.5825884328057473, + "278008524/train2": 0.5825884328057473, + "307068879/train0": 0.87335466614541, + "307068879/train1": 0.87335466614541, + "307068879/train2": 0.87335466614541, + "412102563/train0": 1.0718654827809488, + "412102563/train1": 1.0718654827809488, + "412102563/train2": 1.0718654827809488, + "440577711/train0": 0.8468573198810986, + "440577711/train1": 0.8468573198810986, + "440577711/train2": 0.8468573198810986, + "441046872/train0": 0.87335466614541, + "441046872/train1": 0.87335466614541, + "441046872/train2": 0.87335466614541, + "509430392/train0": 0.8468573198810986, + "509430392/train1": 0.8468573198810986, + "509430392/train2": 0.8468573198810986, + "585904130/train0": 0.5825884328057473, + "585904130/train1": 0.5825884328057473, + "585904130/train2": 0.5825884328057473, + "608016290/train0": 0.87335466614541, + "608016290/train1": 0.87335466614541, + "608016290/train2": 0.87335466614541, + "626486813/train0": 0.5825884328057473, + "626486813/train1": 0.5825884328057473, + "626486813/train2": 0.5825884328057473, + "636680137/train0": 0.5825884328057473, + "636680137/train1": 0.5825884328057473, + "636680137/train2": 0.5825884328057473, + "655884605/train0": 1.0718654827809488, + "655884605/train1": 1.0718654827809488, + "655884605/train2": 1.0718654827809488, + "672328262/train0": 0.5825884328057473, + "672328262/train1": 0.5825884328057473, + "672328262/train2": 0.5825884328057473, + "701667486/train0": 1.0718654827809488, + "701667486/train1": 1.0718654827809488, + "701667486/train2": 1.0718654827809488, + "747260437/train0": 0.5825884328057473, + "747260437/train1": 0.5825884328057473, + "747260437/train2": 0.5825884328057473, + "750448296/train0": 0.5825884328057473, + "750448296/train1": 0.5825884328057473, + "750448296/train2": 0.5825884328057473, + "767747424/train0": 0.5825884328057473, + "767747424/train1": 0.5825884328057473, + "767747424/train2": 0.5825884328057473 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.1192128368353518, + "low": 0.7332976202158582, + "mean": 0.9328310355695862, + "n": 5, + "num_samples": 300 + }, + "mean": 0.9328310355695862, + "per_task": { + "LiftPegUpright-v1": 0.87335466614541, + "PickCube-v1": 1.2894892762347252, + "PullCube-v1": 0.5825884328057477, + "PushCube-v1": 0.8468573198810989, + "StackCube-v1": 1.0718654827809493 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": 0.8080168440864098, + "low": 0.7418667305422796, + "mean": 0.7727632845204696, + "n": 144, + "num_samples": 300 + }, + "n": 144 + } + }, + "target": "success_weighted_margin", + "tau": { + "LiftPegUpright-v1": 0.87335466614541, + "PickCube-v1": 1.2894892762347252, + "PullCube-v1": 0.5825884328057473, + "PushCube-v1": 0.8468573198810986, + "StackCube-v1": 1.0718654827809488, + "__global__": 0.9334263687578273 + }, + "threshold_scope": "task", + "weights": [ + 0.4845035334970376, + 0.06838088727345144, + 0.16678883668530634, + 0.1363771742142653, + -0.1715406741040887, + -0.05724106927282932, + -0.13180922932833544, + -0.4557284010396998, + 0.2706929796541009, + 0.12341385944262218, + 0.12946277027885256, + 0.09769273471009002, + -0.11888136575346309, + -0.11238405821382269, + 0.011849448176968633, + -0.011698642767114149, + -0.11263857629083951, + 0.04832107985993872, + 1.4789877601250618e-14, + -0.6680233898598331, + -0.7755119424783743, + 0.6029408716686788, + 0.8450649960404633, + -0.0803300694570015, + -1.4306019886895758, + -0.45181180635753737, + 0.7706978279797312, + 0.5941190290408258, + 0.23021888951801373, + 0.8402692970746983, + -1.1288945246160321e-14, + 0.5380600170663836, + 0.6962065975153082, + -0.13222563070253313, + -0.22016576761284312, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] +} diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/metrics_by_seed.json b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..d45998d9bea68a48ab05e34d6b01ce7d02316632 --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/metrics_by_seed.json @@ -0,0 +1,3236 @@ +{ + "1073161778/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6134453781512605, + "pairwise_causal_calibration_confidence": 0.5462334122382229, + "pairwise_causal_calibration_ece": 0.06721196591303755, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.8315109279797959, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.87335466614541 + }, + "1073161778/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6833333333333333, + "pairwise_causal_calibration_confidence": 0.5414540870914354, + "pairwise_causal_calibration_ece": 0.14187924624189804, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.696723804413939, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.87335466614541 + }, + "1073161778/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5552324860178256, + "pairwise_causal_calibration_ece": 0.051910371125031565, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.8010166202874374, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.87335466614541 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5542145473380933, + "pairwise_causal_calibration_ece": 0.08992883305237898, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.028473507842246004, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "tau": 1.0718654827809488 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5534020512884718, + "pairwise_causal_calibration_ece": 0.12516937728295682, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.018305243090398104, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "tau": 1.0718654827809488 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5590997153025572, + "pairwise_causal_calibration_ece": 0.11947171326887146, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.16653094820357445, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "tau": 1.0718654827809488 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5428571428571428, + "pairwise_causal_calibration_confidence": 0.5504892981594165, + "pairwise_causal_calibration_ece": 0.007632155302273683, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.8265443858364611, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "tau": 1.0718654827809488 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.38095238095238093, + "pairwise_causal_calibration_confidence": 0.5497312288122934, + "pairwise_causal_calibration_ece": 0.16877884785991246, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.9117051526313378, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3904761904761905, + "pairwise_causal_calibration_confidence": 0.5495148445011372, + "pairwise_causal_calibration_ece": 0.15903865402494677, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.8848141434142904, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "tau": 1.0718654827809488 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.5514987071348044, + "pairwise_causal_calibration_ece": 0.09695325258934998, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.7394668700396011, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4909090909090909, + "pairwise_causal_calibration_confidence": 0.5481788586308872, + "pairwise_causal_calibration_ece": 0.05726976772179637, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.7193134962639322, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5450693806396976, + "pairwise_causal_calibration_ece": 0.0620734765031595, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.7862881124399304, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5765433563593303, + "pairwise_causal_calibration_ece": 0.1479719277879018, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3146091278603853, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "tau": 1.0718654827809488 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5703079922305777, + "pairwise_causal_calibration_ece": 0.07659820557682445, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.05209384637709236, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "tau": 1.0718654827809488 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5728588815635123, + "pairwise_causal_calibration_ece": 0.21571602442065527, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.31434308061981353, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "tau": 1.0718654827809488 + }, + "1324642411/train0": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5666666666666667, + "pairwise_causal_calibration_confidence": 0.5461265642044052, + "pairwise_causal_calibration_ece": 0.020540102462261362, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.8003040878491947, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "selected_candidate_index": 11.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12164360284805298, + "selected_utility_gain_over_base": 0.015173077583312988, + "selector_regret": 0.02241644263267517, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "tau": 0.8468573198810986 + }, + "1324642411/train1": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6218487394957983, + "pairwise_causal_calibration_confidence": 0.5434572117117997, + "pairwise_causal_calibration_ece": 0.07839152778399854, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.3876858687209335, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11658626794815063, + "selected_utility_gain_over_base": 0.010115742683410645, + "selector_regret": 1.8834137320518494, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "1324642411/train2": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.509090909090909, + "pairwise_causal_calibration_confidence": 0.5436846632267275, + "pairwise_causal_calibration_ece": 0.03459375413581838, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.7578621304129007, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.89352947473526, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5866666666666667, + "pairwise_causal_calibration_confidence": 0.5192841401886225, + "pairwise_causal_calibration_ece": 0.06738252647804421, + "pairwise_causal_calibration_pairs": 75.0, + "predicted_margin": 0.13235098862844796, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5386998051668176, + "pairwise_causal_calibration_ece": 0.03869980516681758, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": -0.48662334960024145, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7023809523809523, + "pairwise_causal_calibration_confidence": 0.5315351438061215, + "pairwise_causal_calibration_ece": 0.17084580857483078, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.36262329065526533, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2833333333333333, + "pairwise_causal_calibration_confidence": 0.5448471499959241, + "pairwise_causal_calibration_ece": 0.2615138166625908, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.5415586302800286, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "tau": 0.5825884328057473 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6083333333333333, + "pairwise_causal_calibration_confidence": 0.5656089960064071, + "pairwise_causal_calibration_ece": 0.14136934515423885, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.8524230828537844, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.027254482731223106, + "selected_utility_gain_over_base": -0.07557976432144642, + "selector_regret": 0.009756645187735558, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "tau": 0.5825884328057473 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.425, + "pairwise_causal_calibration_confidence": 0.5603553061579338, + "pairwise_causal_calibration_ece": 0.1353553061579339, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.3015705513281932, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "tau": 0.5825884328057473 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.523550311048989, + "pairwise_causal_calibration_ece": 0.14311635561767766, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.06695685840699794, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "tau": 0.5825884328057473 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3333333333333333, + "pairwise_causal_calibration_confidence": 0.5236824574508954, + "pairwise_causal_calibration_ece": 0.19034912411756205, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.24721792540099097, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "tau": 0.5825884328057473 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5083333333333333, + "pairwise_causal_calibration_confidence": 0.5276472177677529, + "pairwise_causal_calibration_ece": 0.019313884434419615, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.2696456255951446, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006073415279388428, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "tau": 0.5825884328057473 + }, + "1480856810/train0": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8282828282828283, + "pairwise_causal_calibration_confidence": 0.5709539167236203, + "pairwise_causal_calibration_ece": 0.2573289115592079, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 2.2046095259547753, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "1480856810/train1": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5882352941176471, + "pairwise_causal_calibration_confidence": 0.543456481052637, + "pairwise_causal_calibration_ece": 0.04477881306501008, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.735589926228614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "1480856810/train2": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5370429153145398, + "pairwise_causal_calibration_ece": 0.03438565611403155, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.359251321625822, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21894000470638275, + "selected_utility_gain_over_base": 0.1319059282541275, + "selector_regret": 1.7810599952936172, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.5523718802888111, + "pairwise_causal_calibration_ece": 0.10792743584436665, + "pairwise_causal_calibration_pairs": 90.0, + "predicted_margin": 0.6801689818523341, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.19047619047619047, + "pairwise_causal_calibration_confidence": 0.5517045024701317, + "pairwise_causal_calibration_ece": 0.36122831199394134, + "pairwise_causal_calibration_pairs": 63.0, + "predicted_margin": 0.7807195223473116, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5444444444444444, + "pairwise_causal_calibration_confidence": 0.5506485429344871, + "pairwise_causal_calibration_ece": 0.024528598759442233, + "pairwise_causal_calibration_pairs": 90.0, + "predicted_margin": 0.7965612115773528, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.550550217481388, + "pairwise_causal_calibration_ece": 0.11611644918527875, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.3705665707855397, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "tau": 1.2894892762347252 + }, + "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5518834340111771, + "pairwise_causal_calibration_ece": 0.13521676734451044, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.8136424781392986, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "tau": 1.2894892762347252 + }, + "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5635063449255251, + "pairwise_causal_calibration_ece": 0.06840618279050634, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.5686852402790281, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "tau": 1.2894892762347252 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.43333333333333335, + "pairwise_causal_calibration_confidence": 0.5370476153908349, + "pairwise_causal_calibration_ece": 0.10371428205750155, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.332576441483642, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.0347811970859766, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "tau": 0.5825884328057473 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.30833333333333335, + "pairwise_causal_calibration_confidence": 0.5437634159530779, + "pairwise_causal_calibration_ece": 0.2354300826197445, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.9503772340326878, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0303483996540308, + "selected_utility_gain_over_base": -0.07416643761098385, + "selector_regret": 0.16857311688363552, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "tau": 0.5825884328057473 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4583333333333333, + "pairwise_causal_calibration_confidence": 0.540090634882582, + "pairwise_causal_calibration_ece": 0.08175730154924889, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.4498268017490537, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04690708592534065, + "selected_utility_gain_over_base": -0.057607751339673996, + "selector_regret": 0.06619475409388542, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "tau": 0.5825884328057473 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7350427350427351, + "pairwise_causal_calibration_confidence": 0.575799443033985, + "pairwise_causal_calibration_ece": 0.15924329200874998, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": 1.3204376505810713, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.17586350440979, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4090909090909091, + "pairwise_causal_calibration_confidence": 0.5644259196254204, + "pairwise_causal_calibration_ece": 0.15533501053451115, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 1.2385033751871737, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12732218205928802, + "selected_utility_gain_over_base": -0.6968143135309219, + "selector_regret": 1.872677817940712, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5644061750173082, + "pairwise_causal_calibration_ece": 0.13583474644587978, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.1890371636445307, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13359522819519043, + "selected_utility_gain_over_base": -0.6905412673950195, + "selector_regret": 1.8664047718048096, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.6807136864797709, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.7412627528047919, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.7399153467725452, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "1790188787/train0": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2818181818181818, + "pairwise_causal_calibration_confidence": 0.5215209331218725, + "pairwise_causal_calibration_ece": 0.23970275130369073, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 1.1652542261433299, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "1790188787/train1": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3508771929824561, + "pairwise_causal_calibration_confidence": 0.517198535735028, + "pairwise_causal_calibration_ece": 0.16632134275257193, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": 1.1907212928313995, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10585354268550873, + "selected_utility_gain_over_base": -0.01891392469406128, + "selector_regret": 1.8941464573144913, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "1790188787/train2": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.43636363636363634, + "pairwise_causal_calibration_confidence": 0.5206471195621108, + "pairwise_causal_calibration_ece": 0.08428348319847451, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 1.0616754451066321, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1341685652732849, + "selected_utility_gain_over_base": 0.009401097893714905, + "selector_regret": 1.865831434726715, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.575, + "pairwise_causal_calibration_confidence": 0.5678773715560055, + "pairwise_causal_calibration_ece": 0.0719433679479887, + "pairwise_causal_calibration_pairs": 40.0, + "predicted_margin": 0.7418095681624295, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06306678056716919, + "selected_utility_gain_over_base": -0.008848525583744049, + "selector_regret": 1.9369332194328308, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6938775510204082, + "pairwise_causal_calibration_confidence": 0.592873970483461, + "pairwise_causal_calibration_ece": 0.10100358053694718, + "pairwise_causal_calibration_pairs": 49.0, + "predicted_margin": 0.7078283118607503, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9280846938490868, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4, + "pairwise_causal_calibration_confidence": 0.5677066182834786, + "pairwise_causal_calibration_ece": 0.16770661828347844, + "pairwise_causal_calibration_pairs": 40.0, + "predicted_margin": 0.714483922998961, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0730673298239708, + "selected_utility_gain_over_base": 0.0011520236730575562, + "selector_regret": 1.9269326701760292, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "1845489013/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.41304347826086957, + "pairwise_causal_calibration_confidence": 0.51605426349952, + "pairwise_causal_calibration_ece": 0.10301078523865043, + "pairwise_causal_calibration_pairs": 46.0, + "predicted_margin": 0.798908624848506, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "1845489013/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7692307692307693, + "pairwise_causal_calibration_confidence": 0.5252519047235767, + "pairwise_causal_calibration_ece": 0.24397886450719253, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": 0.8663205932771525, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "1845489013/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.72, + "pairwise_causal_calibration_confidence": 0.5274458415050698, + "pairwise_causal_calibration_ece": 0.19255415849493018, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": 0.8229653113797162, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "1999022574/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.543208103565328, + "pairwise_causal_calibration_ece": 0.1860652464224708, + "pairwise_causal_calibration_pairs": 70.0, + "predicted_margin": 0.8291022763943762, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "1999022574/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6714285714285714, + "pairwise_causal_calibration_confidence": 0.5400358084963361, + "pairwise_causal_calibration_ece": 0.13139276293223526, + "pairwise_causal_calibration_pairs": 70.0, + "predicted_margin": 0.7563954831040236, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "1999022574/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3815789473684211, + "pairwise_causal_calibration_confidence": 0.5298970718699219, + "pairwise_causal_calibration_ece": 0.14831812450150075, + "pairwise_causal_calibration_pairs": 76.0, + "predicted_margin": 0.8419564487563823, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.48739495798319327, + "pairwise_causal_calibration_confidence": 0.533600718072752, + "pairwise_causal_calibration_ece": 0.04620576008955871, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.107133532936841, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.713559627532959, + "selected_utility_gain_over_base": -0.01004570722579956, + "selector_regret": 1.286440372467041, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.87335466614541 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6083333333333333, + "pairwise_causal_calibration_confidence": 0.5379143734950628, + "pairwise_causal_calibration_ece": 0.07041895983827043, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.9306280702125403, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7183246612548828, + "selected_utility_gain_over_base": -0.005280673503875732, + "selector_regret": 1.2816753387451172, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.87335466614541 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7717391304347826, + "pairwise_causal_calibration_confidence": 0.5403873973296214, + "pairwise_causal_calibration_ece": 0.2313517331051612, + "pairwise_causal_calibration_pairs": 92.0, + "predicted_margin": 1.1151401969224932, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.2763946652412415, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.87335466614541 + }, + "2038594191/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.27586206896551724, + "pairwise_causal_calibration_confidence": 0.5879562438719218, + "pairwise_causal_calibration_ece": 0.31209417490640445, + "pairwise_causal_calibration_pairs": 29.0, + "predicted_margin": 1.2008765491740636, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "2038594191/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.24074074074074073, + "pairwise_causal_calibration_confidence": 0.6117747139844449, + "pairwise_causal_calibration_ece": 0.37103397324370413, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 1.112157225492381, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07674666494131088, + "selected_utility_gain_over_base": -1.9232533350586891, + "selector_regret": 1.9232533350586891, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "2038594191/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.40404040404040403, + "pairwise_causal_calibration_confidence": 0.5954970669139762, + "pairwise_causal_calibration_ece": 0.19145666287357221, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 1.2364955407573137, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.26666666666666666, + "pairwise_causal_calibration_confidence": 0.5291897348449885, + "pairwise_causal_calibration_ece": 0.13126153408916094, + "pairwise_causal_calibration_pairs": 60.0, + "predicted_margin": 1.014453498377106, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.077283151447773, + "selected_candidate_index": 4.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0643903091549873, + "selected_utility_gain_over_base": -0.9356096908450127, + "selector_regret": 0.012892842292785645, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.922716848552227, + "tau": 0.5825884328057473 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6754385964912281, + "pairwise_causal_calibration_confidence": 0.5263065559429049, + "pairwise_causal_calibration_ece": 0.07456602027416159, + "pairwise_causal_calibration_pairs": 57.0, + "predicted_margin": 1.074027910955616, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5210084033613446, + "pairwise_causal_calibration_confidence": 0.5536973137638624, + "pairwise_causal_calibration_ece": 0.026703688149182295, + "pairwise_causal_calibration_pairs": 59.5, + "predicted_margin": 1.0710380363686038, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.5639694431421344, + "pairwise_causal_calibration_ece": 0.130957329325739, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.39584763737906137, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "tau": 1.0718654827809488 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5581364424191588, + "pairwise_causal_calibration_ece": 0.1414697757524921, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.3705725365164993, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "tau": 1.0718654827809488 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.5794706736680671, + "pairwise_causal_calibration_ece": 0.13502622922362276, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.345599063353308, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "tau": 1.0718654827809488 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6302521008403361, + "pairwise_causal_calibration_confidence": 0.5426955700305087, + "pairwise_causal_calibration_ece": 0.08755653080982741, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.1272780451605109, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8592818975448608, + "selected_utility_gain_over_base": 0.6683734804391861, + "selector_regret": 1.1407181024551392, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.65, + "pairwise_causal_calibration_confidence": 0.5351083386938548, + "pairwise_causal_calibration_ece": 0.12495817852344455, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.2265101367583866, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09285585582256317, + "selected_utility_gain_over_base": -0.09805256128311157, + "selector_regret": 0.12010131776332855, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "tau": 1.0718654827809488 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.525, + "pairwise_causal_calibration_confidence": 0.534871348160612, + "pairwise_causal_calibration_ece": 0.009871348160611883, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.1256343724003985, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13431161642074585, + "selected_utility_gain_over_base": -0.056596800684928894, + "selector_regret": 0.09957115352153778, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "tau": 1.0718654827809488 + }, + "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6191919191919192, + "pairwise_causal_calibration_confidence": 0.5909019399202626, + "pairwise_causal_calibration_ece": 0.1344967059497781, + "pairwise_causal_calibration_pairs": 109.5, + "predicted_margin": 0.27572048526322235, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0733563229441643, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05350120924413204, + "selected_utility_gain_over_base": 0.010766219347715378, + "selector_regret": 1.0198551137000322, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.04055231809616089, + "tau": 0.5825884328057473 + }, + "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5835144927536232, + "pairwise_causal_calibration_confidence": 0.5847988892880861, + "pairwise_causal_calibration_ece": 0.06891187442684295, + "pairwise_causal_calibration_pairs": 106.0, + "predicted_margin": 0.3598249612248885, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0376745946705341, + "selected_candidate_index": 4.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.034389604814350605, + "selected_utility_gain_over_base": -0.008345385082066059, + "selector_regret": 1.0032849898561835, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.07623404636979103, + "tau": 0.5825884328057473 + }, + "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5271464646464646, + "pairwise_causal_calibration_confidence": 0.59445093065676, + "pairwise_causal_calibration_ece": 0.09739801499095443, + "pairwise_causal_calibration_pairs": 109.5, + "predicted_margin": 0.4752736549448759, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1359489113092422, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.047564467415213585, + "selected_utility_gain_over_base": 0.004829477518796921, + "selector_regret": 1.0883844438940287, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5494505494505495, + "pairwise_causal_calibration_confidence": 0.5728850640530763, + "pairwise_causal_calibration_ece": 0.1199981805489268, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": 0.7747416665744656, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "tau": 0.87335466614541 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5274725274725275, + "pairwise_causal_calibration_confidence": 0.5704103700900957, + "pairwise_causal_calibration_ece": 0.07309450817078264, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": 0.3600826916101192, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "tau": 0.87335466614541 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3956043956043956, + "pairwise_causal_calibration_confidence": 0.5723883106472546, + "pairwise_causal_calibration_ece": 0.1767839150428592, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": 0.979654396381198, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2021283358335495, + "selected_utility_gain_over_base": -0.39796222746372223, + "selector_regret": 0.02808511257171631, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "tau": 0.87335466614541 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.65, + "pairwise_causal_calibration_confidence": 0.5493461173759878, + "pairwise_causal_calibration_ece": 0.10951912707521982, + "pairwise_causal_calibration_pairs": 60.0, + "predicted_margin": 1.0435143603129777, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7948717948717948, + "pairwise_causal_calibration_confidence": 0.5593981622164304, + "pairwise_causal_calibration_ece": 0.23547363265536436, + "pairwise_causal_calibration_pairs": 39.0, + "predicted_margin": 0.949568847497014, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4880952380952381, + "pairwise_causal_calibration_confidence": 0.5532244830443317, + "pairwise_causal_calibration_ece": 0.0651292449490937, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.9787258921638226, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "440577711/train0": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6551724137931034, + "pairwise_causal_calibration_confidence": 0.5624630563402808, + "pairwise_causal_calibration_ece": 0.09270935745282255, + "pairwise_causal_calibration_pairs": 29.0, + "predicted_margin": 1.7706684649919293, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "440577711/train1": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5333333333333333, + "pairwise_causal_calibration_confidence": 0.5471459072582541, + "pairwise_causal_calibration_ece": 0.3939678976832891, + "pairwise_causal_calibration_pairs": 15.0, + "predicted_margin": 1.808986225138223, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "440577711/train2": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5185185185185185, + "pairwise_causal_calibration_confidence": 0.5475473443233374, + "pairwise_causal_calibration_ece": 0.17413687717265497, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 1.7310319203842686, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.38181818181818183, + "pairwise_causal_calibration_confidence": 0.5664245191730715, + "pairwise_causal_calibration_ece": 0.19522652344582997, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": 0.6069194020559678, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.3000448346138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.87335466614541 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6181818181818182, + "pairwise_causal_calibration_confidence": 0.5652066435973622, + "pairwise_causal_calibration_ece": 0.09055210003018804, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": 0.1629836409186699, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "tau": 0.87335466614541 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.5755444444938004, + "pairwise_causal_calibration_ece": 0.13114432369738743, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": 0.5214657402479244, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "tau": 0.87335466614541 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6386554621848739, + "pairwise_causal_calibration_confidence": 0.5365694240669033, + "pairwise_causal_calibration_ece": 0.10561970831698192, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.7250273458520244, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11310504376888275, + "selected_utility_gain_over_base": 0.11310482025146484, + "selector_regret": 1.8868949562311172, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5326653211281538, + "pairwise_causal_calibration_ece": 0.038763250300417725, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 1.5703557578982594, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.8468573198810986 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6134453781512605, + "pairwise_causal_calibration_confidence": 0.5403636942457117, + "pairwise_causal_calibration_ece": 0.14212516552457483, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 1.3794654730008236, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "tau": 0.8468573198810986 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.36904761904761907, + "pairwise_causal_calibration_confidence": 0.5472546704459165, + "pairwise_causal_calibration_ece": 0.17820705139829746, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.9073249211388359, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09919407218694687, + "selected_utility_gain_over_base": -0.0825214758515358, + "selector_regret": 1.9008059278130531, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.64, + "pairwise_causal_calibration_confidence": 0.549320518336562, + "pairwise_causal_calibration_ece": 0.19650416069860938, + "pairwise_causal_calibration_pairs": 75.0, + "predicted_margin": 0.8486373501760932, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04792279005050659, + "selected_utility_gain_over_base": -0.13379275798797607, + "selector_regret": 1.9520772099494934, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7272727272727273, + "pairwise_causal_calibration_confidence": 0.5513841945828967, + "pairwise_causal_calibration_ece": 0.1758885326898305, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.9433176675944015, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4857142857142857, + "pairwise_causal_calibration_confidence": 0.594216772727526, + "pairwise_causal_calibration_ece": 0.11412216411454103, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 1.589765155203597, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.707331120967865, + "selected_utility_gain_over_base": -0.018370509147644043, + "selector_regret": 1.292668879032135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.87335466614541 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5047619047619047, + "pairwise_causal_calibration_confidence": 0.5978828235074669, + "pairwise_causal_calibration_ece": 0.20716363895507375, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 1.3392733370703795, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.274298369884491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.87335466614541 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.55, + "pairwise_causal_calibration_confidence": 0.6068596136210345, + "pairwise_causal_calibration_ece": 0.0651220182950748, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.4772393565445312, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6789124011993408, + "selected_utility_gain_over_base": -0.04678922891616821, + "selector_regret": 0.03281521797180176, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "tau": 0.87335466614541 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.65, + "pairwise_causal_calibration_confidence": 0.6158650684827733, + "pairwise_causal_calibration_ece": 0.1759930830571895, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.9601260167395063, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04469547560438514, + "selected_candidate_index": 9.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.007258951431140304, + "selected_utility_gain_over_base": 0.002160608710255474, + "selector_regret": 0.037436524173244834, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12161866808310151, + "tau": 0.5825884328057473 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32916666666666666, + "pairwise_causal_calibration_confidence": 0.6045161568297507, + "pairwise_causal_calibration_ece": 0.2753494901630838, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.9603159694517454, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.035827874671667814, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.011037707328796387, + "selected_utility_gain_over_base": 0.005939364607911557, + "selector_regret": 0.024790167342871428, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12975450651720166, + "tau": 0.5825884328057473 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.625, + "pairwise_causal_calibration_confidence": 0.6142120016040145, + "pairwise_causal_calibration_ece": 0.14970168842483308, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.7735840442684991, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04489925503730774, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.012973218690603971, + "selected_utility_gain_over_base": 0.007874875969719142, + "selector_regret": 0.03192603634670377, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12068312615156174, + "tau": 0.5825884328057473 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.425, + "pairwise_causal_calibration_confidence": 0.540894445846163, + "pairwise_causal_calibration_ece": 0.11589444584616293, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 1.0758545824030454, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.056092143058776855, + "selected_utility_gain_over_base": -0.027042903006076813, + "selector_regret": 0.0321691632270813, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "tau": 0.5825884328057473 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.625, + "pairwise_causal_calibration_confidence": 0.5301042033378723, + "pairwise_causal_calibration_ece": 0.09489579666212766, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.44432064636845636, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "tau": 0.5825884328057473 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.475, + "pairwise_causal_calibration_confidence": 0.5238615170185813, + "pairwise_causal_calibration_ece": 0.048861517018581324, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.5548863156073045, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "tau": 0.5825884328057473 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5833333333333334, + "pairwise_causal_calibration_confidence": 0.5958635862373126, + "pairwise_causal_calibration_ece": 0.10068633489228643, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.5292851601669687, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "tau": 1.0718654827809488 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5277777777777778, + "pairwise_causal_calibration_confidence": 0.6117746577593474, + "pairwise_causal_calibration_ece": 0.228982599451425, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.8854182759065323, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6498691290616989, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "tau": 1.0718654827809488 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4722222222222222, + "pairwise_causal_calibration_confidence": 0.5899703889806722, + "pairwise_causal_calibration_ece": 0.19094254082377798, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.7967317370820184, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "tau": 1.0718654827809488 + }, + "672328262/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8235294117647058, + "pairwise_causal_calibration_confidence": 0.5940991028072462, + "pairwise_causal_calibration_ece": 0.22943030895745967, + "pairwise_causal_calibration_pairs": 17.0, + "predicted_margin": 0.7871393331842744, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "672328262/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5428571428571428, + "pairwise_causal_calibration_confidence": 0.5924504110795782, + "pairwise_causal_calibration_ece": 0.07756824536937998, + "pairwise_causal_calibration_pairs": 35.0, + "predicted_margin": 0.7852289914727505, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "672328262/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2, + "pairwise_causal_calibration_confidence": 0.6012678269199632, + "pairwise_causal_calibration_ece": 0.4012678269199631, + "pairwise_causal_calibration_pairs": 30.0, + "predicted_margin": 0.6365332388231544, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04013325646519661, + "selected_utility_gain_over_base": -1.9598667435348034, + "selector_regret": 1.9598667435348034, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5825884328057473 + }, + "701667486/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.38095238095238093, + "pairwise_causal_calibration_confidence": 0.5278746131288938, + "pairwise_causal_calibration_ece": 0.1469222321765129, + "pairwise_causal_calibration_pairs": 42.0, + "predicted_margin": 1.1356123561511042, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "701667486/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5308683756394741, + "pairwise_causal_calibration_ece": 0.004845910074811617, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 1.234980143424118, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "701667486/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.5331361711412006, + "pairwise_causal_calibration_ece": 0.08869172669675614, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 1.3518684532465013, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8125, + "selected_utility_gain_over_base": -1.1875, + "selector_regret": 1.1875, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 1.0718654827809488 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5083333333333333, + "pairwise_causal_calibration_confidence": 0.5322967641018285, + "pairwise_causal_calibration_ece": 0.046545342019779196, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.7403769985406177, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07023122161626816, + "selected_utility_gain_over_base": -1.9297687783837318, + "selector_regret": 0.0381644144654274, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "tau": 0.5825884328057473 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.675, + "pairwise_causal_calibration_confidence": 0.5502411276617256, + "pairwise_causal_calibration_ece": 0.2890079292574854, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.6210780034951271, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.05181918665766716, + "selected_utility_gain_over_base": -1.9481808133423328, + "selector_regret": 0.011470336467027664, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "tau": 0.5825884328057473 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7083333333333334, + "pairwise_causal_calibration_confidence": 0.5279460992108118, + "pairwise_causal_calibration_ece": 0.1803872341225216, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.9133990308397049, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.0880771279335022, + "selected_utility_gain_over_base": -1.9119228720664978, + "selector_regret": 0.03702475130558014, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "tau": 0.5825884328057473 + }, + "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.5793650793650794, + "pairwise_causal_calibration_confidence": 0.5359906885657441, + "pairwise_causal_calibration_ece": 0.07322390346905903, + "pairwise_causal_calibration_pairs": 108.0, + "predicted_margin": 0.8003581361777009, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 0.7526603279014429, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09101381525397301, + "selected_utility_gain_over_base": -0.026579334090153377, + "selector_regret": 0.6616465126474699, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6553361676633358, + "tau": 0.5825884328057473 + }, + "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4423076923076923, + "pairwise_causal_calibration_confidence": 0.5377836086909143, + "pairwise_causal_calibration_ece": 0.09632988234770806, + "pairwise_causal_calibration_pairs": 101.66666666666667, + "predicted_margin": 0.4055514323654445, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3705032741030057, + "selected_candidate_index": 7.333333333333333, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10720233743389447, + "selected_utility_gain_over_base": -0.010390811910231909, + "selector_regret": 1.2633009366691113, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.03749322146177292, + "tau": 0.5825884328057473 + }, + "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.4833333333333334, + "pairwise_causal_calibration_confidence": 0.5376337500179175, + "pairwise_causal_calibration_ece": 0.07076889063044275, + "pairwise_causal_calibration_pairs": 108.0, + "predicted_margin": 0.5039790420368024, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3504486680030823, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09631064782540004, + "selected_utility_gain_over_base": -0.02128250151872635, + "selector_regret": 1.2541380201776822, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.05754782756169637, + "tau": 0.5825884328057473 + }, + "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "pairwise_causal_calibration_accuracy": 0.49059829059829063, + "pairwise_causal_calibration_confidence": 0.5592639661990686, + "pairwise_causal_calibration_ece": 0.1274080019549793, + "pairwise_causal_calibration_pairs": 119.25, + "predicted_margin": 0.5457716259676435, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0424291878007352, + "selected_candidate_index": 8.25, + "selected_success": 0.0, + "selected_success_gain_over_base": -0.25, + "selected_utility": 0.0617859915655572, + "selected_utility_gain_over_base": -0.5114133504393976, + "selector_regret": 0.980643196235178, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.06562284985557199, + "tau": 0.5825884328057473 + }, + "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "pairwise_causal_calibration_accuracy": 0.5700877145730087, + "pairwise_causal_calibration_confidence": 0.5526263680260839, + "pairwise_causal_calibration_ece": 0.15613421435842795, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.6028818781511358, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0629530912265182, + "selected_candidate_index": 3.75, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5621845133136958, + "selected_utility_gain_over_base": -0.011014828691259027, + "selector_regret": 0.5007685779128224, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.045098946429789066, + "tau": 0.5825884328057473 + }, + "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "pairwise_causal_calibration_accuracy": 0.5193142641672053, + "pairwise_causal_calibration_confidence": 0.5607639712421639, + "pairwise_causal_calibration_ece": 0.10791269448114733, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.5655051629030958, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0671670949086547, + "selected_candidate_index": 8.75, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.567537541501224, + "selected_utility_gain_over_base": -0.005661800503730774, + "selector_regret": 0.49962955340743065, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.04088494274765253, + "tau": 0.5825884328057473 + } +} diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/metrics_by_task.json b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..73cc20399067d1c250b60848cc36429977b72b70 --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/metrics_by_task.json @@ -0,0 +1,137 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.2, + "base_utility": 0.9498705387115478, + "coverage": 0.4666666666666667, + "execute_generated": 0.4666666666666667, + "fallback_rate": 0.5333333333333333, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.5333333333333333, + "pairwise_causal_calibration_accuracy": 0.5492625405285252, + "pairwise_causal_calibration_confidence": 0.5651094024103738, + "pairwise_causal_calibration_ece": 0.11881236057424142, + "pairwise_causal_calibration_pairs": 94.8, + "predicted_margin": 0.8862852359573267, + "proposal_oracle_success": 0.6, + "proposal_oracle_utility": 1.4122399787108104, + "selected_candidate_index": 5.333333333333333, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.13333333333333333, + "selected_utility": 1.088020184636116, + "selected_utility_gain_over_base": 0.1381496459245682, + "selector_regret": 0.3509542107582092, + "success_selector_gap": 0.26666666666666666, + "success_support_gap": 0.4, + "support_gap": 0.5877600212891897, + "tau": 0.87335466614541 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5277777777777778, + "pairwise_causal_calibration_confidence": 0.5553133321393634, + "pairwise_causal_calibration_ece": 0.10657979977343184, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.5842980964012888, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13425676027933756, + "selected_candidate_index": 7.333333333333333, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8657432397206624, + "tau": 1.2894892762347252 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.7751704612234608, + "coverage": 0.6533333333333333, + "execute_generated": 0.6533333333333333, + "fallback_rate": 0.3466666666666667, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.4666666666666667, + "pairwise_causal_calibration_accuracy": 0.5216143238307843, + "pairwise_causal_calibration_confidence": 0.5584521161129652, + "pairwise_causal_calibration_ece": 0.13244853449538532, + "pairwise_causal_calibration_pairs": 92.32, + "predicted_margin": 0.6759761060180944, + "proposal_oracle_success": 0.5333333333333333, + "proposal_oracle_utility": 1.1093241602430741, + "selected_candidate_index": 5.986666666666666, + "selected_success": 0.29333333333333333, + "selected_success_gain_over_base": -0.06666666666666667, + "selected_utility": 0.6341658942727372, + "selected_utility_gain_over_base": -0.14100456695072353, + "selector_regret": 0.4787018513850247, + "success_selector_gap": 0.24, + "success_support_gap": 0.02666666666666667, + "support_gap": 0.1075624164690574, + "tau": 0.5825884328057477 + }, + "PushCube-v1": { + "base_success": 0.16666666666666666, + "base_utility": 0.3924041787783305, + "coverage": 0.7777777777777778, + "execute_generated": 0.7777777777777778, + "fallback_rate": 0.2222222222222222, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.8333333333333334, + "pairwise_causal_calibration_accuracy": 0.5465423944585421, + "pairwise_causal_calibration_confidence": 0.5419772241344875, + "pairwise_causal_calibration_ece": 0.11799694975778399, + "pairwise_causal_calibration_pairs": 97.44444444444444, + "predicted_margin": 1.3829754161601442, + "proposal_oracle_success": 0.8888888888888888, + "proposal_oracle_utility": 1.7857814149724112, + "selected_candidate_index": 3.7222222222222223, + "selected_success": 0.5555555555555556, + "selected_success_gain_over_base": 0.3888888888888889, + "selected_utility": 1.162042656706439, + "selected_utility_gain_over_base": 0.7696384779281087, + "selector_regret": 0.6237387582659721, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.1111111111111111, + "support_gap": 0.21421858502758873, + "tau": 0.8468573198810989 + }, + "StackCube-v1": { + "base_success": 0.2727272727272727, + "base_utility": 0.7763947045261209, + "coverage": 0.2727272727272727, + "execute_generated": 0.2727272727272727, + "fallback_rate": 0.7272727272727273, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.6363636363636364, + "pairwise_causal_calibration_accuracy": 0.5043423274143793, + "pairwise_causal_calibration_confidence": 0.5567990222574275, + "pairwise_causal_calibration_ece": 0.13094581844275163, + "pairwise_causal_calibration_pairs": 67.45454545454545, + "predicted_margin": 0.7229237046506615, + "proposal_oracle_success": 0.5151515151515151, + "proposal_oracle_utility": 1.1567951707225856, + "selected_candidate_index": 5.393939393939394, + "selected_success": 0.2727272727272727, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7495684027671814, + "selected_utility_gain_over_base": -0.02682630175893957, + "selector_regret": 0.41081975310137775, + "success_selector_gap": 0.24242424242424243, + "success_support_gap": 0.48484848484848486, + "support_gap": 0.8432048292774142, + "tau": 1.0718654827809493 + } +} diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/split_hash.txt b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..865ad41e869c1bfbc57e4db65ed79e91e6d93081 --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/split_hash.txt @@ -0,0 +1,14 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + } +} diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/table.tex b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..7186b70d786ad05e70fc59fa214c5be42dd4e19c --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_learned_dominance_selector.py +\begin{tabular}{lrrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap & Cal. ECE \\ +\midrule +144 & 0.5486 & 0.4514 & 0.2917 & 0.3194 & 0.5694 & 0.5486 & 0.2014 & 0.2500 & 0.0302 \\ +\bottomrule +\end{tabular} diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/train.log b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..1c48597a347313250c9c105b1c05dfeda6ec0094 --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_successbonus2_task_envclip_k16_train_to_test/train.log @@ -0,0 +1,5 @@ +trained ridge dominance calibrator on calibration measured rows only +fit_objective=pointwise +selected_lambda=10.0 +tau={"LiftPegUpright-v1": 0.873355, "PickCube-v1": 1.289489, "PullCube-v1": 0.582588, "PushCube-v1": 0.846857, "StackCube-v1": 1.071865, "__global__": 0.933426} +calibration_input=runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json