diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/command.txt b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..e7f0b8fb12329026484da8a0f2dc093cb5e295da --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_learned_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --source-index data/cil_charts_rgb_refs/train/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test --feature-set score_chart_compat --selector-chart-feature-mode base_context_obs --target utility_margin --threshold-scope task --k 16 --bootstrap-samples 300 --no-markdown-report diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/config.yaml b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..810ed38bee4cdf48416dc0659577d8e76fa94be3 --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/config.yaml @@ -0,0 +1,18 @@ +bootstrap_samples: 300 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: score_chart_compat +fit_objective: pointwise +k: 16 +no_markdown_report: True +out_dir: runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test +pairwise_weight: 1.0 +ridge_lambdas: 0,0.01,0.1,1,10,100 +selector_chart_feature_mode: base_context_obs +source_index: data/cil_charts_rgb_refs/train/index.json +success_bonus: 1.0 +target: utility_margin +threshold_scope: task diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/data_hash.txt b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..e65c158c17e61aaf10422563d2ffdd8922735e6a --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/data_hash.txt @@ -0,0 +1,7 @@ +{ + "calibration_input": "d79e275b896eb98caa128470d4614b76960cab12407faa157c583224be7abe80", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "541d5a928da5f7ee04ca3dea570e04208830dffde431edfe5b5e38b692634045", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36", + "source_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85" +} diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/eval.log b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..ad44bd37e51f82471f7a17d7372cee5fed00ed3f --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/eval.log @@ -0,0 +1,3 @@ +evaluated learned fallback rule on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/git_hash.txt b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/metrics.json b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..47cc3be4121e30ba7ec7ee8c29ce9ead998df45c --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/metrics.json @@ -0,0 +1,9008 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "calibration_model_selection": { + "base_success": 0.2777777777777778, + "base_utility": 0.7007981554626086, + "coverage": 0.6481481481481481, + "fallback_rate": 0.35185185185185186, + "hidden_chart_oracle_success": 0.6597222222222222, + "hidden_chart_oracle_utility": 1.4150809271054134, + "outcome_ptr": 0.5462962962962963, + "proposal_oracle_success": 0.5370370370370371, + "proposal_oracle_utility": 1.1440672337592803, + "selected_success": 0.3611111111111111, + "selected_success_gain_over_base": 0.08333333333333333, + "selected_utility": 0.8304847452566236, + "selector_regret": 0.40203449528825674, + "success_selector_gap": 0.2175925925925926, + "success_support_gap": 0.17592592592592593, + "support_gap": 0.35465237537627453 + }, + "calibration_summary": { + "base_success": 0.2777777777777778, + "base_utility": 0.7007981554626086, + "coverage": 0.6481481481481481, + "fallback_rate": 0.35185185185185186, + "hidden_chart_oracle_success": 0.6597222222222222, + "hidden_chart_oracle_utility": 1.4150809271054134, + "outcome_ptr": 0.5462962962962963, + "pairwise_causal_calibration_accuracy": 0.5151666754436779, + "pairwise_causal_calibration_confidence": 0.5305006974422358, + "pairwise_causal_calibration_ece": 0.015334021998557847, + "pairwise_causal_calibration_pairs": 37978.0, + "proposal_oracle_success": 0.5370370370370371, + "proposal_oracle_utility": 1.1440672337592803, + "selected_success": 0.3611111111111111, + "selected_success_gain_over_base": 0.08333333333333333, + "selected_utility": 0.8304847452566236, + "selector_regret": 0.40203449528825674, + "success_selector_gap": 0.2175925925925926, + "success_support_gap": 0.17592592592592593, + "support_gap": 0.35465237537627453 + }, + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.2916666666666667, + "base_utility": 0.7418195864423373, + "coverage": 0.6388888888888888, + "fallback_rate": 0.3611111111111111, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.5486111111111112, + "pairwise_causal_calibration_accuracy": 0.5346630207495576, + "pairwise_causal_calibration_confidence": 0.5293185199263176, + "pairwise_causal_calibration_ece": 0.022107332086325975, + "pairwise_causal_calibration_pairs": 12434.0, + "proposal_oracle_success": 0.5694444444444444, + "proposal_oracle_utility": 1.2159999172436073, + "selected_success": 0.3263888888888889, + "selected_success_gain_over_base": 0.034722222222222224, + "selected_utility": 0.7742872366474735, + "selector_regret": 0.4654409392969683, + "success_selector_gap": 0.25, + "success_support_gap": 0.2013888888888889, + "support_gap": 0.3761285081272945 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_mean": [ + 0.0, + 4.602051421289207, + 1.8887871984592688, + -1.985099232970326e-17, + 7.187557603686636, + 0.0663594470046083, + 2.0903225806451613, + 0.7930022866760579, + 1.9819565879721794, + 15.375115207373272, + 0.5, + 0.06635944700460818, + -1.1461792655128946, + -0.8539997540940796, + -0.16156509362744845, + 0.16156509362744845, + 0.5331797235023041, + 0.29217951141881504, + 1.0, + 1.044349800585736, + 1.0509106460665045, + 0.8002031215841607, + 0.5811250768510506, + 2.5434513049028507, + 0.34326541665726357, + 0.8966401857535892, + 1.2182893520781068, + 1.2263923089918751, + 0.786961656617233, + 0.6987682628846764, + 1.0, + 0.9974076898437442, + 0.02008356847617045, + 0.39137378168005876, + 0.3910654018162905, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "candidate_score_rank_fraction", + "candidate_score_softmax_prob", + "candidate_score_gap_to_best", + "candidate_score_gap_to_second_best", + "candidate_score_gap_to_prev_higher", + "candidate_score_gap_to_next_lower", + "candidate_score_percentile", + "candidate_score_top_margin", + "source_chart_feature_found", + "target_chart_feature_norm", + "source_chart_feature_norm", + "target_source_chart_cosine", + "target_source_chart_rms", + "target_source_chart_linf", + "target_source_chart_absmean", + "target_source_chart_dot_per_dim", + "target_base_action_norm", + "source_base_action_norm", + "target_source_base_cosine", + "target_source_base_rms", + "target_source_obs_available", + "target_source_obs_cosine", + "target_source_obs_rms", + "target_obs_norm", + "source_obs_norm", + "target_source_obj_available", + "target_source_obj_cosine", + "target_source_obj_rms", + "target_obj_norm", + "source_obj_norm" + ], + "feature_set": "score_chart_compat", + "feature_std": [ + 1.0, + 2.5232884736710055, + 2.2444575887945213, + 0.9999992484766103, + 4.542613580634851, + 0.24891036261589178, + 1.9642314545113713, + 0.37078681697003224, + 0.7171840796063713, + 1.7485561930736828, + 0.30856036102426265, + 0.04867152161828175, + 0.9290082113338436, + 0.8520345362910837, + 0.21685697548301888, + 0.21685697548301888, + 0.2880797360439561, + 0.30393911054414136, + 1e-06, + 0.14319094131293478, + 0.14491309378768996, + 0.19298897178584146, + 0.32495613500512616, + 1.3413619446954763, + 0.2065176097090355, + 0.3376678168884184, + 0.1766334065498482, + 0.1790096989966568, + 0.20687067948130008, + 0.3907071033474156, + 1e-06, + 0.005866370796541541, + 0.02376663855951178, + 0.015525196749693172, + 0.017227249939028074, + 1e-06, + 1e-06, + 1e-06, + 1e-06, + 1e-06 + ], + "fit_design": { + "fit_objective": "pointwise", + "num_candidate_rows": 6510, + "num_fit_rows": 6510, + "num_pairwise_rows": 0, + "num_pointwise_rows": 6510, + "pairwise_weight": 0.0 + }, + "fit_objective": "pointwise", + "k": 16, + "num_calibration_candidates": 6510, + "num_calibration_rows": 432, + "num_eval_candidates": 2115, + "num_eval_rows": 144, + "pairwise_causal_calibration": { + "calibration": { + "accuracy": 0.5151666754436779, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.01165962380424257, + "accuracy": 0.5152193497311021, + "confidence": 0.5268789735353446, + "count": 36631, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.09968388489006574, + "accuracy": 0.525908739365816, + "confidence": 0.6255926242558817, + "count": 1293, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.4881582356512103, + "accuracy": 0.2222222222222222, + "confidence": 0.7103804578734325, + "count": 54, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.015334021998557847, + "mean_confidence": 0.5305006974422358, + "n_bins": 10, + "num_pairs": 37978, + "num_rows": 432 + }, + "eval": { + "accuracy": 0.5346630207495576, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.014323797330992205, + "accuracy": 0.5394880402853546, + "confidence": 0.5251642429543624, + "count": 11915, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.2007986935695666, + "accuracy": 0.4238921001926782, + "confidence": 0.6246907937622448, + "count": 519, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.022107332086325975, + "mean_confidence": 0.5293185199263176, + "n_bins": 10, + "num_pairs": 12434, + "num_rows": 144 + } + }, + "pairwise_weight": 1.0, + "report_type": "learned_dominance_selector_eval", + "ridge_lambdas": [ + 0.0, + 0.01, + 0.1, + 1.0, + 10.0, + 100.0 + ], + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3949579831932773, + "pairwise_causal_calibration_confidence": 0.5282333871650522, + "pairwise_causal_calibration_ece": 0.1332754039717749, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.1490312481371141, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train0", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7026570439338684, + "selected_utility_gain_over_base": -0.020948290824890137, + "selector_regret": 1.2973429560661316, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5494505494505495, + "pairwise_causal_calibration_confidence": 0.5260813668824347, + "pairwise_causal_calibration_ece": 0.036558379935610945, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": 0.08783512114247624, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "seed": "307068879/train0", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19761089980602264, + "selected_utility_gain_over_base": -0.4024796634912491, + "selector_regret": 0.01394665241241455, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6218487394957983, + "pairwise_causal_calibration_confidence": 0.5252122988397412, + "pairwise_causal_calibration_ece": 0.09663644065605714, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.07106166187087547, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.67014479637146, + "selected_utility_gain_over_base": -1.32985520362854, + "selector_regret": 1.32985520362854, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5619047619047619, + "pairwise_causal_calibration_confidence": 0.5561217648468477, + "pairwise_causal_calibration_ece": 0.03209834914225741, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.5011206164338899, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train0", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.274298369884491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.36363636363636365, + "pairwise_causal_calibration_confidence": 0.5267148410723766, + "pairwise_causal_calibration_ece": 0.163078477436013, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": 0.046645784157056724, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "441046872/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20773810148239136, + "selected_utility_gain_over_base": -0.4922170639038086, + "selector_regret": 1.7922618985176086, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5358486568758676, + "pairwise_causal_calibration_ece": 0.03584865687586758, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": -0.37628277603589594, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "seed": "1522272686/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "task_id": "PickCube-v1", + "tau": 0.5025584143631503, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.04624259356089375, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.38461538461538464, + "pairwise_causal_calibration_confidence": 0.5087871241212454, + "pairwise_causal_calibration_ece": 0.12417173950586075, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": -0.19362564784596037, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02627839706838131, + "selected_utility_gain_over_base": -0.13540753163397312, + "selector_regret": 1.9737216029316187, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.30833333333333335, + "pairwise_causal_calibration_confidence": 0.5145099591738125, + "pairwise_causal_calibration_ece": 0.20617662584047913, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.09082588792911983, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "seed": "1461069713/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.014822185039520264, + "selected_utility_gain_over_base": -0.07908087968826294, + "selector_regret": 0.04537618160247803, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7416666666666667, + "pairwise_causal_calibration_confidence": 0.5757476456290026, + "pairwise_causal_calibration_ece": 0.22293034880133372, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.775200972367557, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.010183374397456646, + "seed": "626486813/train0", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.007048348430544138, + "selected_utility_gain_over_base": 0.005118846776895225, + "selector_regret": 0.003135025966912508, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24323733616620302, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.26666666666666666, + "pairwise_causal_calibration_confidence": 0.5290379603266726, + "pairwise_causal_calibration_ece": 0.26237129366000594, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.02174125236488622, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15456630289554596, + "seed": "2054899126/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.08063507080078125, + "selected_utility_gain_over_base": -1.9193649291992188, + "selector_regret": 0.07393123209476471, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.845433697104454, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8235294117647058, + "pairwise_causal_calibration_confidence": 0.5525242267379454, + "pairwise_causal_calibration_ece": 0.27100518502676063, + "pairwise_causal_calibration_pairs": 17.0, + "predicted_margin": -0.087075237393225, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4583333333333333, + "pairwise_causal_calibration_confidence": 0.556288803981115, + "pairwise_causal_calibration_ece": 0.09795547064778175, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.160531410420454, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15051491558551788, + "seed": "767747424/train0", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08540759235620499, + "selected_utility_gain_over_base": 0.00841909646987915, + "selector_regret": 0.0651073232293129, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08458571135997772, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6916666666666667, + "pairwise_causal_calibration_confidence": 0.5246693313972304, + "pairwise_causal_calibration_ece": 0.16699733526943622, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.33009554566017457, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14671264588832855, + "seed": "278008524/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05741874501109123, + "selected_utility_gain_over_base": 0.021532438695430756, + "selector_regret": 0.08929390087723732, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08110463619232178, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32608695652173914, + "pairwise_causal_calibration_confidence": 0.5077729807205135, + "pairwise_causal_calibration_ece": 0.18168602419877433, + "pairwise_causal_calibration_pairs": 46.0, + "predicted_margin": -0.22001414669968217, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5666666666666667, + "pairwise_causal_calibration_confidence": 0.5556290097511218, + "pairwise_causal_calibration_ece": 0.011037656915544881, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.2681415236496052, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07920757681131363, + "seed": "626486813/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.012751738540828228, + "selected_utility_gain_over_base": 0.004484554752707481, + "selector_regret": 0.0664558382704854, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7166666666666667, + "pairwise_causal_calibration_confidence": 0.5143600301618059, + "pairwise_causal_calibration_ece": 0.2023066365048608, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.31217491974788636, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5296207411344719, + "pairwise_causal_calibration_ece": 0.06533502684875758, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.10499618981526784, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09919407218694687, + "selected_utility_gain_over_base": -0.0825214758515358, + "selector_regret": 1.9008059278130531, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.523230296021853, + "pairwise_causal_calibration_ece": 0.10656362935518632, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.08363360102690726, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "seed": "141984615/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.007176061626523733, + "selected_utility_gain_over_base": -0.09565818542614579, + "selector_regret": 0.008001843933016062, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.49166666666666664, + "pairwise_causal_calibration_confidence": 0.5256123951340491, + "pairwise_causal_calibration_ece": 0.033945728467382474, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.018465700329916374, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21658658981323242, + "seed": "750448296/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21658658981323242, + "selected_utility_gain_over_base": 0.07387763261795044, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7834134101867676, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5190311348285646, + "pairwise_causal_calibration_ece": 0.10236446816189787, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.4324165282764497, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "seed": "636680137/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.056092143058776855, + "selected_utility_gain_over_base": -0.027042903006076813, + "selector_regret": 0.0321691632270813, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6333333333333333, + "pairwise_causal_calibration_confidence": 0.534999768197681, + "pairwise_causal_calibration_ece": 0.09833356513565228, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.23052873356798131, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.019201835617423058, + "seed": "767747424/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01041970681399107, + "selected_utility_gain_over_base": -0.04370323661714792, + "selector_regret": 0.008782128803431988, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17790568806231022, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5289748634459925, + "pairwise_causal_calibration_ece": 0.1376918032206741, + "pairwise_causal_calibration_pairs": 75.0, + "predicted_margin": -0.29978209413001944, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.27586206896551724, + "pairwise_causal_calibration_confidence": 0.5740754232851301, + "pairwise_causal_calibration_ece": 0.2982133543196128, + "pairwise_causal_calibration_pairs": 29.0, + "predicted_margin": 0.19762473284895746, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04285776615142822, + "selected_utility_gain_over_base": -1.9571422338485718, + "selector_regret": 1.9571422338485718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6166666666666667, + "pairwise_causal_calibration_confidence": 0.5120242051814605, + "pairwise_causal_calibration_ece": 0.10464246148520617, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.33932393024086993, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.041394393891096115, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.041394393891096115, + "selected_utility_gain_over_base": -0.05160309746861458, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18259509280323982, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.32706425588430094, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.49166666666666664, + "pairwise_causal_calibration_confidence": 0.5222884856095076, + "pairwise_causal_calibration_ece": 0.03062181894284094, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.08999271197759756, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "seed": "747260437/train0", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.061248183250427246, + "selected_utility_gain_over_base": -1.9387518167495728, + "selector_regret": 0.04714745283126831, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.5404725232935694, + "pairwise_causal_calibration_ece": 0.08592706874811483, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.11381592128190807, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 14, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9504163265228271, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5117256505009358, + "pairwise_causal_calibration_ece": 0.15494101616573086, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.495453729546493, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "seed": "1549324805/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.0347811970859766, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.575, + "pairwise_causal_calibration_confidence": 0.549416493223813, + "pairwise_causal_calibration_ece": 0.08571587301053908, + "pairwise_causal_calibration_pairs": 40.0, + "predicted_margin": -0.0008074765493970848, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06247224286198616, + "selected_utility_gain_over_base": -0.009443063288927078, + "selector_regret": 1.9375277571380138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.40476190476190477, + "pairwise_causal_calibration_confidence": 0.5246521315903555, + "pairwise_causal_calibration_ece": 0.11989022682845074, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.3401193749830937, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6206896551724138, + "pairwise_causal_calibration_confidence": 0.5273358125858131, + "pairwise_causal_calibration_ece": 0.09335384258660073, + "pairwise_causal_calibration_pairs": 29.0, + "predicted_margin": 0.5465320030087847, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5583333333333333, + "pairwise_causal_calibration_confidence": 0.5085746791327035, + "pairwise_causal_calibration_ece": 0.04975865420062986, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.5936226389105781, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "seed": "1324642411/train0", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11045536398887634, + "selected_utility_gain_over_base": 0.0039848387241363525, + "selector_regret": 0.03360468149185181, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.38181818181818183, + "pairwise_causal_calibration_confidence": 0.5077769577736889, + "pairwise_causal_calibration_ece": 0.12595877595550709, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.1484177313105067, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.5174492210423474, + "pairwise_causal_calibration_ece": 0.06290376649689283, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": -0.08116638752440412, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6134453781512605, + "pairwise_causal_calibration_confidence": 0.5129229068912616, + "pairwise_causal_calibration_ece": 0.10052247125999891, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.4269940390592024, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11310504376888275, + "selected_utility_gain_over_base": 0.11310482025146484, + "selector_regret": 1.8868949562311172, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8383838383838383, + "pairwise_causal_calibration_confidence": 0.5267941522147715, + "pairwise_causal_calibration_ece": 0.3115896861690668, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.8247860886762404, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5393032283831621, + "pairwise_causal_calibration_ece": 0.06094260950628667, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.12372212373443373, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "seed": "2061635888/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5277777777777778, + "pairwise_causal_calibration_confidence": 0.5597351518736766, + "pairwise_causal_calibration_ece": 0.05616248877449882, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.07276876911827704, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "seed": "655884605/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5381830912240237, + "pairwise_causal_calibration_ece": 0.10961166265259512, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1171582058773368, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "seed": "1113371261/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7008547008547008, + "pairwise_causal_calibration_confidence": 0.5507814264578395, + "pairwise_causal_calibration_ece": 0.15007327439686133, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": 0.2626760081803321, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train0", + "selected_candidate_index": 9, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.17586350440979, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.43333333333333335, + "pairwise_causal_calibration_confidence": 0.5388105224755593, + "pairwise_causal_calibration_ece": 0.10547718914222602, + "pairwise_causal_calibration_pairs": 90.0, + "predicted_margin": -0.11725750852840032, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5882352941176471, + "pairwise_causal_calibration_confidence": 0.517501300906042, + "pairwise_causal_calibration_ece": 0.07073399321160512, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.04424689933210735, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "211245842/train0", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8090915828943253, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5142857142857142, + "pairwise_causal_calibration_confidence": 0.5296341103177351, + "pairwise_causal_calibration_ece": 0.015348396032020872, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.12464332364425626, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "seed": "1151410346/train0", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5557404501598795, + "pairwise_causal_calibration_ece": 0.16288330730273656, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.09117531897647907, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "seed": "1324133122/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5238095238095238, + "pairwise_causal_calibration_confidence": 0.5307788011897017, + "pairwise_causal_calibration_ece": 0.00696927738017783, + "pairwise_causal_calibration_pairs": 42.0, + "predicted_margin": 0.18126309408722283, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train0", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5444159724840614, + "pairwise_causal_calibration_ece": 0.04441597248406137, + "pairwise_causal_calibration_pairs": 60.0, + "predicted_margin": 0.0942295054093702, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2571428571428571, + "pairwise_causal_calibration_confidence": 0.5358621206887306, + "pairwise_causal_calibration_ece": 0.2787192635458735, + "pairwise_causal_calibration_pairs": 70.0, + "predicted_margin": 0.0012762962711883319, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.575, + "pairwise_causal_calibration_confidence": 0.5289841828268461, + "pairwise_causal_calibration_ece": 0.04601581717315384, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.036105644427698036, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5218750448469349, + "pairwise_causal_calibration_ece": 0.049553526581636476, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": -0.13881394667704933, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "seed": "307068879/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6083333333333333, + "pairwise_causal_calibration_confidence": 0.5228008383368301, + "pairwise_causal_calibration_ece": 0.08553249499650317, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.05584757187568534, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.49523809523809526, + "pairwise_causal_calibration_confidence": 0.5555371517280593, + "pairwise_causal_calibration_ece": 0.06029905648996403, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.2910589829678913, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.274298369884491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6909090909090909, + "pairwise_causal_calibration_confidence": 0.5234185829111347, + "pairwise_causal_calibration_ece": 0.16749050799795617, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": -0.17358234568930467, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "seed": "441046872/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2222222222222222, + "pairwise_causal_calibration_confidence": 0.5423563498452709, + "pairwise_causal_calibration_ece": 0.32013412762304877, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": -0.18697224010199912, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "seed": "1522272686/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "task_id": "PickCube-v1", + "tau": 0.5025584143631503, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.00797935515478132, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.36134453781512604, + "pairwise_causal_calibration_confidence": 0.5104506142141808, + "pairwise_causal_calibration_ece": 0.14910607639905477, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": -0.33419524839225123, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8383140712976456, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5141448492443679, + "pairwise_causal_calibration_ece": 0.09747818257770119, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.1449663100493438, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "seed": "1461069713/train1", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.006729066371917725, + "selected_utility_gain_over_base": -0.08717399835586548, + "selector_regret": 0.009855946525931358, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.325, + "pairwise_causal_calibration_confidence": 0.5716128728942771, + "pairwise_causal_calibration_ece": 0.256947612057456, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.6113998221493578, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.009895960800349712, + "seed": "626486813/train1", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00289076566696167, + "selected_utility_gain_over_base": 0.000961264013312757, + "selector_regret": 0.0070051951333880424, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24352474976330996, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6491228070175439, + "pairwise_causal_calibration_confidence": 0.5248824145830566, + "pairwise_causal_calibration_ece": 0.12424039243448726, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": -0.1555929299105951, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.45714285714285713, + "pairwise_causal_calibration_confidence": 0.5406287558110948, + "pairwise_causal_calibration_ece": 0.08348589866823782, + "pairwise_causal_calibration_pairs": 35.0, + "predicted_margin": -0.23567605508015027, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.5525406960296153, + "pairwise_causal_calibration_ece": 0.05187229094802172, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.14222716429755156, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20903043448925018, + "seed": "767747424/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18194365501403809, + "selected_utility_gain_over_base": 0.10495515912771225, + "selector_regret": 0.027086779475212097, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026070192456245422, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4083333333333333, + "pairwise_causal_calibration_confidence": 0.5179096505561795, + "pairwise_causal_calibration_ece": 0.10957631722284622, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.47002328115376657, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07534918934106827, + "seed": "278008524/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01919553615152836, + "selected_utility_gain_over_base": -0.016690770164132118, + "selector_regret": 0.05615365318953991, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15246809273958206, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 1.0, + "pairwise_causal_calibration_confidence": 0.5209647285753305, + "pairwise_causal_calibration_ece": 0.4790352714246695, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": -0.18822734864823312, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.425, + "pairwise_causal_calibration_confidence": 0.5534969698356498, + "pairwise_causal_calibration_ece": 0.1284969698356498, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.3519937608406678, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.061759788542985916, + "seed": "626486813/train1", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.010010083205997944, + "selected_utility_gain_over_base": 0.0017428994178771973, + "selector_regret": 0.05174970533698797, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01598426327109337, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7606837606837606, + "pairwise_causal_calibration_confidence": 0.5106997024711076, + "pairwise_causal_calibration_ece": 0.24998405821265302, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": -0.16627136796176625, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.5273196708342051, + "pairwise_causal_calibration_ece": 0.07268032916579492, + "pairwise_causal_calibration_pairs": 75.0, + "predicted_margin": 0.06533471439439086, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train1", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04792279005050659, + "selected_utility_gain_over_base": -0.13379275798797607, + "selector_regret": 1.9520772099494934, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.575, + "pairwise_causal_calibration_confidence": 0.5259787015164451, + "pairwise_causal_calibration_ece": 0.049021298483554854, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.1560213376803143, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "seed": "141984615/train1", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.030119141563773155, + "selected_utility_gain_over_base": -0.07271510548889637, + "selector_regret": 0.006891986355185509, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.522212848434421, + "pairwise_causal_calibration_ece": 0.22778715156557905, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.30627797855821914, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7, + "pairwise_causal_calibration_confidence": 0.5160986674159838, + "pairwise_causal_calibration_ece": 0.1839013325840162, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.0489088109005007, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "seed": "636680137/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.017789265140891075, + "selected_utility_gain_over_base": -0.0653457809239626, + "selector_regret": 0.04962191917002201, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6333333333333333, + "pairwise_causal_calibration_confidence": 0.5313942238827244, + "pairwise_causal_calibration_ece": 0.10193910945060891, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.30496918903874165, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.042781930416822433, + "seed": "767747424/train1", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.042781930416822433, + "selected_utility_gain_over_base": -0.011341013014316559, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15432559326291084, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5614035087719298, + "pairwise_causal_calibration_confidence": 0.5257629988292399, + "pairwise_causal_calibration_ece": 0.03564050994268986, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": -0.79693146060142, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2962962962962963, + "pairwise_causal_calibration_confidence": 0.5667318952152414, + "pairwise_causal_calibration_ece": 0.27043559891894486, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 0.3209231912001711, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07674666494131088, + "selected_utility_gain_over_base": -1.9232533350586891, + "selector_regret": 1.9232533350586891, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.375, + "pairwise_causal_calibration_confidence": 0.5124292478571429, + "pairwise_causal_calibration_ece": 0.13742924785714294, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.1084374964792017, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11150982230901718, + "seed": "750448296/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06907258182764053, + "selected_utility_gain_over_base": -0.02392490953207016, + "selector_regret": 0.04243724048137665, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.11247966438531876, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.4986531018645557, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6833333333333333, + "pairwise_causal_calibration_confidence": 0.5252521749807111, + "pairwise_causal_calibration_ece": 0.15808115835262226, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.26165809689080527, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "seed": "747260437/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6413043478260869, + "pairwise_causal_calibration_confidence": 0.5465697715124626, + "pairwise_causal_calibration_ece": 0.09473457631362422, + "pairwise_causal_calibration_pairs": 92.0, + "predicted_margin": 0.09876914550533503, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9504163265228271, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.575, + "pairwise_causal_calibration_confidence": 0.5062538341615629, + "pairwise_causal_calibration_ece": 0.06874616583843707, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.18733344993267076, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "seed": "1549324805/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0303483996540308, + "selected_utility_gain_over_base": -0.07416643761098385, + "selector_regret": 0.16857311688363552, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6530612244897959, + "pairwise_causal_calibration_confidence": 0.557604964366195, + "pairwise_causal_calibration_ece": 0.09545626012360084, + "pairwise_causal_calibration_pairs": 49.0, + "predicted_margin": -0.0650108915156051, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.045472364872694016, + "selected_utility_gain_over_base": -0.026442941278219223, + "selector_regret": 1.954527635127306, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.47692307692307695, + "pairwise_causal_calibration_confidence": 0.5243799390305641, + "pairwise_causal_calibration_ece": 0.047456862107487174, + "pairwise_causal_calibration_pairs": 65.0, + "predicted_margin": -0.39646879322768863, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4, + "pairwise_causal_calibration_confidence": 0.5197720209730798, + "pairwise_causal_calibration_ece": 0.11977202097307982, + "pairwise_causal_calibration_pairs": 15.0, + "predicted_margin": 0.629918599045802, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.680672268907563, + "pairwise_causal_calibration_confidence": 0.5079157579147822, + "pairwise_causal_calibration_ece": 0.1727565109927809, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.3040303172517523, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11658626794815063, + "selected_utility_gain_over_base": 0.010115742683410645, + "selector_regret": 1.8834137320518494, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.37719298245614036, + "pairwise_causal_calibration_confidence": 0.5066944209005328, + "pairwise_causal_calibration_ece": 0.1295014384443925, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": 0.13132582655221522, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10585354268550873, + "selected_utility_gain_over_base": -0.01891392469406128, + "selector_regret": 1.8941464573144913, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5272727272727272, + "pairwise_causal_calibration_confidence": 0.5193562339693999, + "pairwise_causal_calibration_ece": 0.007916493303327332, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": -0.09547952648462638, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5619047619047619, + "pairwise_causal_calibration_confidence": 0.5125583130541626, + "pairwise_causal_calibration_ece": 0.049346448850599356, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.37833433181770104, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5798319327731093, + "pairwise_causal_calibration_confidence": 0.5180113719626723, + "pairwise_causal_calibration_ece": 0.061820560810436964, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.5725138287112228, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21981161832809448, + "selected_utility_gain_over_base": 0.13277754187583923, + "selector_regret": 1.7801883816719055, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4722222222222222, + "pairwise_causal_calibration_confidence": 0.5362308522692728, + "pairwise_causal_calibration_ece": 0.06400863004705062, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.049366979703384384, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "seed": "2061635888/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.562343415271088, + "pairwise_causal_calibration_ece": 0.11789897082664356, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.1554209538300596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "seed": "655884605/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11176733672618866, + "selected_utility_gain_over_base": -0.051239341497421265, + "selector_regret": 0.7011084705591202, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5386619397445, + "pairwise_causal_calibration_ece": 0.06848091739835714, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.11952381111596475, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "seed": "1113371261/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42727272727272725, + "pairwise_causal_calibration_confidence": 0.5448159860797186, + "pairwise_causal_calibration_ece": 0.11754325880699122, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.17364935131576656, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12732218205928802, + "selected_utility_gain_over_base": -0.6968143135309219, + "selector_regret": 1.872677817940712, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.23809523809523808, + "pairwise_causal_calibration_confidence": 0.5371878138277117, + "pairwise_causal_calibration_ece": 0.29909257573247366, + "pairwise_causal_calibration_pairs": 63.0, + "predicted_margin": -0.10799597937369632, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6583333333333333, + "pairwise_causal_calibration_confidence": 0.5160911960797463, + "pairwise_causal_calibration_ece": 0.14224213725358703, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.13077847063672782, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "seed": "211245842/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02204875648021698, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.44761904761904764, + "pairwise_causal_calibration_confidence": 0.528839557722319, + "pairwise_causal_calibration_ece": 0.08122051010327141, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.10435665617161227, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1151410346/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5544807305695781, + "pairwise_causal_calibration_ece": 0.10154169690861004, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.10108263691246488, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "seed": "1324133122/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5952380952380952, + "pairwise_causal_calibration_confidence": 0.5325935443428202, + "pairwise_causal_calibration_ece": 0.06264455089527499, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.15700516615012938, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train1", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7692307692307693, + "pairwise_causal_calibration_confidence": 0.5435121854300012, + "pairwise_causal_calibration_ece": 0.22571858380076806, + "pairwise_causal_calibration_pairs": 39.0, + "predicted_margin": 0.029952749201257664, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5297587479178145, + "pairwise_causal_calibration_ece": 0.041669823510756876, + "pairwise_causal_calibration_pairs": 70.0, + "predicted_margin": -0.08747088123670077, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6847826086956522, + "pairwise_causal_calibration_confidence": 0.531948622673026, + "pairwise_causal_calibration_ece": 0.1528339860226262, + "pairwise_causal_calibration_pairs": 92.0, + "predicted_margin": 0.09375170906247592, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train2", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6934840679168701, + "selected_utility_gain_over_base": -0.030121266841888428, + "selector_regret": 1.3065159320831299, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4945054945054945, + "pairwise_causal_calibration_confidence": 0.5232228710238422, + "pairwise_causal_calibration_ece": 0.028717376518347715, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": 0.14397102567158404, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "seed": "307068879/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22035211324691772, + "selected_utility_gain_over_base": -0.379738450050354, + "selector_regret": 0.009861335158348083, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5254991602652951, + "pairwise_causal_calibration_ece": 0.14116750640137155, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.07873777798305576, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5083333333333333, + "pairwise_causal_calibration_confidence": 0.5609759280259827, + "pairwise_causal_calibration_ece": 0.05264259469264974, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.4536901796255097, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "seed": "608016290/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6789124011993408, + "selected_utility_gain_over_base": -0.04678922891616821, + "selector_regret": 0.03281521797180176, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.5255994659745129, + "pairwise_causal_calibration_ece": 0.07105401142905837, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": -0.022380468606574055, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "seed": "441046872/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "task_id": "LiftPegUpright-v1", + "tau": 0.028235521236802175, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5411589331210354, + "pairwise_causal_calibration_ece": 0.12449226645436867, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": -0.2402513599376978, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "seed": "1522272686/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "task_id": "PickCube-v1", + "tau": 0.5025584143631503, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.06617060088497922, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 15, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6638655462184874, + "pairwise_causal_calibration_confidence": 0.5138205036707686, + "pairwise_causal_calibration_ece": 0.15004504254771878, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": -0.24604226210470226, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8383140712976456, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.5185093923599849, + "pairwise_causal_calibration_ece": 0.08149060764001503, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.13654486820462297, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "seed": "1461069713/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09997648000717163, + "selected_utility_gain_over_base": 0.006073415279388428, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7583333333333333, + "pairwise_causal_calibration_confidence": 0.5718595242801999, + "pairwise_causal_calibration_ece": 0.18647380905313332, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.5453619418295237, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03885193541646004, + "seed": "626486813/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.017679253593087196, + "selected_utility_gain_over_base": 0.015749751939438283, + "selector_regret": 0.02117268182337284, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21456877514719963, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5798319327731093, + "pairwise_causal_calibration_confidence": 0.5308674006096046, + "pairwise_causal_calibration_ece": 0.048964532163504715, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": -0.16497563292480844, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 14, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4, + "pairwise_causal_calibration_confidence": 0.5442903896362881, + "pairwise_causal_calibration_ece": 0.1442903896362882, + "pairwise_causal_calibration_pairs": 30.0, + "predicted_margin": -0.14512362014342817, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04013325646519661, + "selected_utility_gain_over_base": -1.9598667435348034, + "selector_regret": 1.9598667435348034, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3, + "pairwise_causal_calibration_confidence": 0.5580073082495534, + "pairwise_causal_calibration_ece": 0.2580073082495535, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.11968923816243313, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21886132657527924, + "seed": "767747424/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0730469822883606, + "selected_utility_gain_over_base": -0.0039415135979652405, + "selector_regret": 0.14581434428691864, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01623930037021637, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.528326612006066, + "pairwise_causal_calibration_ece": 0.11165994533939946, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.2601575872438925, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2718978226184845, + "seed": "278008524/train2", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04554526135325432, + "selected_utility_gain_over_base": 0.009658955037593842, + "selector_regret": 0.22635256126523018, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.44, + "pairwise_causal_calibration_confidence": 0.5128055424573941, + "pairwise_causal_calibration_ece": 0.07280554245739407, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -0.1547908021155039, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train2", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.525, + "pairwise_causal_calibration_confidence": 0.5554467028317209, + "pairwise_causal_calibration_ece": 0.030446702831720726, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.23546795238077922, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05094657465815544, + "seed": "626486813/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.016867956146597862, + "selected_utility_gain_over_base": 0.008600772358477116, + "selector_regret": 0.03407861851155758, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026797477155923843, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5213675213675214, + "pairwise_causal_calibration_confidence": 0.509457332297461, + "pairwise_causal_calibration_ece": 0.011910189070060362, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": -0.2428803998963185, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7818181818181819, + "pairwise_causal_calibration_confidence": 0.529609307543077, + "pairwise_causal_calibration_ece": 0.25220887427510486, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.09915967492706912, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5333333333333333, + "pairwise_causal_calibration_confidence": 0.5258876236158011, + "pairwise_causal_calibration_ece": 0.007445709717532223, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.046015598870960614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "seed": "141984615/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02716982364654541, + "selected_utility_gain_over_base": -0.07566442340612411, + "selector_regret": 0.031781893223524094, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7166666666666667, + "pairwise_causal_calibration_confidence": 0.5298039058264857, + "pairwise_causal_calibration_ece": 0.18686276084018094, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.0072536998424092974, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12231997400522232, + "selected_utility_gain_over_base": -0.020388983190059662, + "selector_regret": 1.8776800259947777, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5129089770472632, + "pairwise_causal_calibration_ece": 0.012908977047263237, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.13337534819821673, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "seed": "636680137/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.015303333289921284, + "selected_utility_gain_over_base": -0.06783171277493238, + "selector_regret": 0.060348290018737316, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6166666666666667, + "pairwise_causal_calibration_confidence": 0.5411620972822431, + "pairwise_causal_calibration_ece": 0.1266896226987229, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.46716305154776483, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04980705305933952, + "seed": "767747424/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04339444637298584, + "selected_utility_gain_over_base": -0.010728497058153152, + "selector_regret": 0.0064126066863536835, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14730047062039375, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8095238095238095, + "pairwise_causal_calibration_confidence": 0.5337083842579382, + "pairwise_causal_calibration_ece": 0.27581542526587133, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.6377766658396278, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.5635629372227472, + "pairwise_causal_calibration_ece": 0.1090174826772927, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.31779873234459755, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.12977690994739532, + "selected_utility_gain_over_base": -1.8702230900526047, + "selector_regret": 1.8702230900526047, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5061085398457259, + "pairwise_causal_calibration_ece": 0.006108539845725902, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.10476507994192752, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.051346004009246826, + "seed": "750448296/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035547394305467606, + "selected_utility_gain_over_base": -0.05745009705424309, + "selector_regret": 0.01579860970377922, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1726434826850891, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.4919042736821675, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.5187728818008999, + "pairwise_causal_calibration_ece": 0.08122711819910011, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.061796090652556146, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "seed": "747260437/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06438694149255753, + "selected_utility_gain_over_base": -1.9356130585074425, + "selector_regret": 0.06071493774652481, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6262626262626263, + "pairwise_causal_calibration_confidence": 0.543680207968306, + "pairwise_causal_calibration_ece": 0.1027583191032813, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.22274532006538736, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 15, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9504163265228271, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.575, + "pairwise_causal_calibration_confidence": 0.5094928991923253, + "pairwise_causal_calibration_ece": 0.06550710080767463, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.3994404854817541, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "seed": "1549324805/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04690708592534065, + "selected_utility_gain_over_base": -0.057607751339673996, + "selector_regret": 0.06619475409388542, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.425, + "pairwise_causal_calibration_confidence": 0.5487264271441268, + "pairwise_causal_calibration_ece": 0.12372642714412677, + "pairwise_causal_calibration_pairs": 40.0, + "predicted_margin": -0.0006465763829108795, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08447941392660141, + "selected_utility_gain_over_base": 0.012564107775688171, + "selector_regret": 1.9155205860733986, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.40476190476190477, + "pairwise_causal_calibration_confidence": 0.5308387675806765, + "pairwise_causal_calibration_ece": 0.12607686281877173, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.563486512521856, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.2012384152043573, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.46296296296296297, + "pairwise_causal_calibration_confidence": 0.5240762582462818, + "pairwise_causal_calibration_ece": 0.061113295283318825, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 0.6241310075694073, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 13, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5181818181818182, + "pairwise_causal_calibration_confidence": 0.5077976736149897, + "pairwise_causal_calibration_ece": 0.010384144566828457, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.07036169250445845, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.89352947473526, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.38181818181818183, + "pairwise_causal_calibration_confidence": 0.5083204445849703, + "pairwise_causal_calibration_ece": 0.12650226276678844, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.12795113455003831, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10904359817504883, + "selected_utility_gain_over_base": -0.01572386920452118, + "selector_regret": 1.8909564018249512, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6190476190476191, + "pairwise_causal_calibration_confidence": 0.5192244871616464, + "pairwise_causal_calibration_ece": 0.0998231318859727, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.0006632054109868557, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5966386554621849, + "pairwise_causal_calibration_confidence": 0.5151111613972951, + "pairwise_causal_calibration_ece": 0.0815274940648898, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.282290647599035, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "seed": "509430392/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5630252100840336, + "pairwise_causal_calibration_confidence": 0.5179749462983955, + "pairwise_causal_calibration_ece": 0.045050263785638145, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.4285300550683504, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21894000470638275, + "selected_utility_gain_over_base": 0.1319059282541275, + "selector_regret": 1.7810599952936172, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.07420081675594419, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5391540330154272, + "pairwise_causal_calibration_ece": 0.016401522540128385, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.015355053119340732, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "seed": "2061635888/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5588372328026256, + "pairwise_causal_calibration_ece": 0.027075654701186606, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.11907170730895571, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "seed": "655884605/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5353470658344741, + "pairwise_causal_calibration_ece": 0.036081505594097285, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.2072636199490521, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "seed": "1113371261/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.36134453781512604, + "pairwise_causal_calibration_confidence": 0.5460279556731249, + "pairwise_causal_calibration_ece": 0.18468341785799877, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.15403622504955924, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15726424753665924, + "selected_utility_gain_over_base": -0.6668722480535507, + "selector_regret": 1.8427357524633408, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4888888888888889, + "pairwise_causal_calibration_confidence": 0.5391151823956642, + "pairwise_causal_calibration_ece": 0.05022629350677536, + "pairwise_causal_calibration_pairs": 90.0, + "predicted_margin": -0.17204314625253667, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train2", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5666666666666667, + "pairwise_causal_calibration_confidence": 0.5172535124316141, + "pairwise_causal_calibration_ece": 0.04941315423505255, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.04386283650419778, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "seed": "211245842/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04297435283660889, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.38095238095238093, + "pairwise_causal_calibration_confidence": 0.530463392824977, + "pairwise_causal_calibration_ece": 0.14951101187259608, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.0965746708031926, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "seed": "1151410346/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5529109212469531, + "pairwise_causal_calibration_ece": 0.33862520696123877, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.05985917108029659, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "seed": "1324133122/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5185185185185185, + "pairwise_causal_calibration_confidence": 0.5303820261796057, + "pairwise_causal_calibration_ece": 0.011863507661087236, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 0.1922641707596195, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.38095238095238093, + "pairwise_causal_calibration_confidence": 0.5454433115811214, + "pairwise_causal_calibration_ece": 0.1644909306287403, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.004657759279381279, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42105263157894735, + "pairwise_causal_calibration_confidence": 0.5267734463002417, + "pairwise_causal_calibration_ece": 0.10572081472129435, + "pairwise_causal_calibration_pairs": 76.0, + "predicted_margin": -0.10186397965031571, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.13235082030966006, + "train_seed": "2" + } + ], + "schema_version": 1, + "selected_lambda": 100.0, + "selector_calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "selector_chart_feature_mode": "base_context_obs", + "selector_eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "selector_eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "selector_source_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "success_bonus": 1.0, + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38958333333333334, + "low": 0.22083333333333333, + "mean": 0.30625, + "n": 120, + "num_samples": 300 + }, + "mean": 0.30625, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3038787878787878, + "low": 0.09454545454545454, + "mean": 0.19987878787878788, + "n": 5, + "num_samples": 300 + }, + "mean": 0.19987878787878788, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.3680555555555556, + "low": 0.2152777777777778, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9549208872815751, + "low": 0.6291768466153169, + "mean": 0.7901180622977944, + "n": 120, + "num_samples": 300 + }, + "mean": 0.7901180622977944, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.12476746737957001, + "1790188787/train1": 0.12476746737957001, + "1790188787/train2": 0.12476746737957001, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 2.2351741790771484e-07, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.00509834272088483, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.11759314934412639, + "750448296/train1": 0.11759314934412639, + "750448296/train2": 0.11759314934412639, + "767747424/train0": 0.5731993420049548, + "767747424/train1": 0.5731993420049548, + "767747424/train2": 0.5731993420049548 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.879990507716313, + "low": 0.5455794048082704, + "mean": 0.695559797174015, + "n": 5, + "num_samples": 300 + }, + "mean": 0.695559797174015, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7751704612234608, + "PushCube-v1": 0.3924041787783305, + "StackCube-v1": 0.7763947045261209 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.8711436447313948, + "low": 0.6016125785051069, + "mean": 0.7418195864423373, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6993055555555557, + "low": 0.5277777777777778, + "mean": 0.6138888888888889, + "n": 120, + "num_samples": 300 + }, + "mean": 0.6138888888888889, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 0.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 1.0, + "307068879/train1": 0.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 0.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 0.0, + "747260437/train2": 1.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.75, + "767747424/train1": 0.75, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.784, + "low": 0.20509090909090907, + "mean": 0.4939797979797979, + "n": 5, + "num_samples": 300 + }, + "mean": 0.4939797979797979, + "per_task": { + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.8133333333333334, + "PushCube-v1": 0.7777777777777778, + "StackCube-v1": 0.21212121212121213 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.7152777777777778, + "low": 0.5625, + "mean": 0.6388888888888888, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6993055555555557, + "low": 0.5277777777777778, + "mean": 0.6138888888888889, + "n": 120, + "num_samples": 300 + }, + "mean": 0.6138888888888889, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 0.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 1.0, + "307068879/train1": 0.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 0.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 0.0, + "747260437/train2": 1.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.75, + "767747424/train1": 0.75, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.784, + "low": 0.20509090909090907, + "mean": 0.4939797979797979, + "n": 5, + "num_samples": 300 + }, + "mean": 0.4939797979797979, + "per_task": { + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.8133333333333334, + "PushCube-v1": 0.7777777777777778, + "StackCube-v1": 0.21212121212121213 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.7152777777777778, + "low": 0.5625, + "mean": 0.6388888888888888, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4722222222222222, + "low": 0.3006944444444445, + "mean": 0.3861111111111111, + "n": 120, + "num_samples": 300 + }, + "mean": 0.3861111111111111, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 1.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 0.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 1.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7949090909090909, + "low": 0.21600000000000003, + "mean": 0.506020202020202, + "n": 5, + "num_samples": 300 + }, + "mean": 0.506020202020202, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 1.0, + "PullCube-v1": 0.18666666666666668, + "PushCube-v1": 0.2222222222222222, + "StackCube-v1": 0.7878787878787878 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.4375, + "low": 0.2847222222222222, + "mean": 0.3611111111111111, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8597222222222223, + "low": 0.7097222222222221, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 300 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 300 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7487430048278638, + "low": 1.487454852130678, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 300 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 300 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.6414945029343169, + "low": 1.3965875044878986, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6180555555555555, + "low": 0.44861111111111107, + "mean": 0.5319444444444444, + "n": 120, + "num_samples": 300 + }, + "mean": 0.5319444444444444, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 1.0, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7151515151515151, + "low": 0.23393939393939395, + "mean": 0.49393939393939396, + "n": 5, + "num_samples": 300 + }, + "mean": 0.49393939393939396, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4666666666666667, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.6363636363636364 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.6388888888888888, + "low": 0.4722222222222222, + "mean": 0.5486111111111112, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "pairwise_causal_calibration_accuracy": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5513360014629635, + "low": 0.4987694224782799, + "mean": 0.5238405556765252, + "n": 117, + "num_samples": 300 + }, + "mean": 0.5238405556765252, + "per_seed": { + "1073161778/train0": 0.6218487394957983, + "1073161778/train1": 0.6083333333333333, + "1073161778/train2": 0.6666666666666666, + "1113371261/train0": 0.42857142857142855, + "1113371261/train1": 0.6071428571428571, + "1113371261/train2": 0.5714285714285714, + "1151410346/train0": 0.5142857142857142, + "1151410346/train1": 0.44761904761904764, + "1151410346/train2": 0.38095238095238093, + "1261930456/train0": 0.45454545454545453, + "1261930456/train1": 0.5272727272727272, + "1261930456/train2": 0.6190476190476191, + "1324133122/train0": 0.39285714285714285, + "1324133122/train1": 0.4642857142857143, + "1324133122/train2": 0.21428571428571427, + "1324642411/train0": 0.5583333333333333, + "1324642411/train1": 0.680672268907563, + "1324642411/train2": 0.5181818181818182, + "1375492786/train0": 0.6666666666666666, + "1375492786/train1": 0.5614035087719298, + "1375492786/train2": 0.8095238095238095, + "141984615/train0": 0.4166666666666667, + "141984615/train1": 0.575, + "141984615/train2": 0.5333333333333333, + "1461069713/train0": 0.30833333333333335, + "1461069713/train1": 0.4166666666666667, + "1461069713/train2": 0.6, + "1480856810/train0": 0.8383838383838383, + "1480856810/train1": 0.5798319327731093, + "1480856810/train2": 0.5630252100840336, + "1509980664/train0": 0.43333333333333335, + "1509980664/train1": 0.23809523809523808, + "1509980664/train2": 0.4888888888888889, + "1522272686/train0": 0.5, + "1522272686/train1": 0.2222222222222222, + "1522272686/train2": 0.4166666666666667, + "1549324805/train0": 0.6666666666666666, + "1549324805/train1": 0.575, + "1549324805/train2": 0.575, + "1578990937/train0": 0.7008547008547008, + "1578990937/train1": 0.42727272727272725, + "1578990937/train2": 0.36134453781512604, + "1790188787/train0": 0.38181818181818183, + "1790188787/train1": 0.37719298245614036, + "1790188787/train2": 0.38181818181818183, + "1822210471/train0": 0.575, + "1822210471/train1": 0.6530612244897959, + "1822210471/train2": 0.425, + "1845489013/train0": 0.32608695652173914, + "1845489013/train1": 1.0, + "1845489013/train2": 0.44, + "1999022574/train0": 0.2571428571428571, + "1999022574/train1": 0.5714285714285714, + "1999022574/train2": 0.42105263157894735, + "2012211287/train0": 0.3949579831932773, + "2012211287/train1": 0.575, + "2012211287/train2": 0.6847826086956522, + "2038594191/train0": 0.27586206896551724, + "2038594191/train1": 0.2962962962962963, + "2038594191/train2": 0.45454545454545453, + "2054899126/train0": 0.26666666666666666, + "2054899126/train1": 0.6491228070175439, + "2054899126/train2": 0.5798319327731093, + "2061635888/train0": 0.5, + "2061635888/train1": 0.4722222222222222, + "2061635888/train2": 0.5555555555555556, + "211245842/train0": 0.5882352941176471, + "211245842/train1": 0.6583333333333333, + "211245842/train2": 0.5666666666666667, + "278008524/train0": 0.5731060606060606, + "278008524/train1": 0.5248188405797101, + "278008524/train2": 0.5214646464646465, + "307068879/train0": 0.5494505494505495, + "307068879/train1": 0.5714285714285714, + "307068879/train2": 0.4945054945054945, + "412102563/train0": 0.5, + "412102563/train1": 0.7692307692307693, + "412102563/train2": 0.38095238095238093, + "440577711/train0": 0.6206896551724138, + "440577711/train1": 0.4, + "440577711/train2": 0.46296296296296297, + "441046872/train0": 0.36363636363636365, + "441046872/train1": 0.6909090909090909, + "441046872/train2": 0.45454545454545453, + "509430392/train0": 0.6134453781512605, + "509430392/train1": 0.5619047619047619, + "509430392/train2": 0.5966386554621849, + "585904130/train0": 0.4642857142857143, + "585904130/train1": 0.6, + "585904130/train2": 0.7818181818181819, + "608016290/train0": 0.5619047619047619, + "608016290/train1": 0.49523809523809526, + "608016290/train2": 0.5083333333333333, + "626486813/train0": 0.6541666666666667, + "626486813/train1": 0.375, + "626486813/train2": 0.6416666666666666, + "636680137/train0": 0.4166666666666667, + "636680137/train1": 0.7, + "636680137/train2": 0.5, + "655884605/train0": 0.5277777777777778, + "655884605/train1": 0.4444444444444444, + "655884605/train2": 0.5555555555555556, + "672328262/train0": 0.8235294117647058, + "672328262/train1": 0.45714285714285713, + "672328262/train2": 0.4, + "701667486/train0": 0.5238095238095238, + "701667486/train1": 0.5952380952380952, + "701667486/train2": 0.5185185185185185, + "747260437/train0": 0.49166666666666664, + "747260437/train1": 0.6833333333333333, + "747260437/train2": 0.6, + "750448296/train0": 0.5043650793650793, + "750448296/train1": 0.533974358974359, + "750448296/train2": 0.5404761904761904, + "767747424/train0": 0.5482371794871794, + "767747424/train1": 0.5888404079580549, + "767747424/train2": 0.5254749335631689 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5455222296456455, + "low": 0.43510367170021186, + "mean": 0.5002133038437834, + "n": 5, + "num_samples": 300 + }, + "mean": 0.5002133038437834, + "per_task": { + "LiftPegUpright-v1": 0.5494360697557629, + "PickCube-v1": 0.3796296296296296, + "PullCube-v1": 0.5439315887395845, + "PushCube-v1": 0.5408758312375324, + "StackCube-v1": 0.4871933998564077 + } + }, + "metric": "pairwise_causal_calibration_accuracy", + "micro": { + "confidence": 0.95, + "high": 0.5488772820166298, + "low": 0.503951158120747, + "mean": 0.5269917153318382, + "n": 138, + "num_samples": 300 + }, + "n": 138 + }, + "pairwise_causal_calibration_confidence": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5346481921937558, + "low": 0.5285073781330066, + "mean": 0.5313682932910728, + "n": 117, + "num_samples": 300 + }, + "mean": 0.5313682932910728, + "per_seed": { + "1073161778/train0": 0.5252122988397412, + "1073161778/train1": 0.5228008383368301, + "1073161778/train2": 0.5254991602652951, + "1113371261/train0": 0.5381830912240237, + "1113371261/train1": 0.5386619397445, + "1113371261/train2": 0.5353470658344741, + "1151410346/train0": 0.5296341103177351, + "1151410346/train1": 0.528839557722319, + "1151410346/train2": 0.530463392824977, + "1261930456/train0": 0.5174492210423474, + "1261930456/train1": 0.5193562339693999, + "1261930456/train2": 0.5192244871616464, + "1324133122/train0": 0.5557404501598795, + "1324133122/train1": 0.5544807305695781, + "1324133122/train2": 0.5529109212469531, + "1324642411/train0": 0.5085746791327035, + "1324642411/train1": 0.5079157579147822, + "1324642411/train2": 0.5077976736149897, + "1375492786/train0": 0.5289748634459925, + "1375492786/train1": 0.5257629988292399, + "1375492786/train2": 0.5337083842579382, + "141984615/train0": 0.523230296021853, + "141984615/train1": 0.5259787015164451, + "141984615/train2": 0.5258876236158011, + "1461069713/train0": 0.5145099591738125, + "1461069713/train1": 0.5141448492443679, + "1461069713/train2": 0.5185093923599849, + "1480856810/train0": 0.5267941522147715, + "1480856810/train1": 0.5180113719626723, + "1480856810/train2": 0.5179749462983955, + "1509980664/train0": 0.5388105224755593, + "1509980664/train1": 0.5371878138277117, + "1509980664/train2": 0.5391151823956642, + "1522272686/train0": 0.5358486568758676, + "1522272686/train1": 0.5423563498452709, + "1522272686/train2": 0.5411589331210354, + "1549324805/train0": 0.5117256505009358, + "1549324805/train1": 0.5062538341615629, + "1549324805/train2": 0.5094928991923253, + "1578990937/train0": 0.5507814264578395, + "1578990937/train1": 0.5448159860797186, + "1578990937/train2": 0.5460279556731249, + "1790188787/train0": 0.5077769577736889, + "1790188787/train1": 0.5066944209005328, + "1790188787/train2": 0.5083204445849703, + "1822210471/train0": 0.549416493223813, + "1822210471/train1": 0.557604964366195, + "1822210471/train2": 0.5487264271441268, + "1845489013/train0": 0.5077729807205135, + "1845489013/train1": 0.5209647285753305, + "1845489013/train2": 0.5128055424573941, + "1999022574/train0": 0.5358621206887306, + "1999022574/train1": 0.5297587479178145, + "1999022574/train2": 0.5267734463002417, + "2012211287/train0": 0.5282333871650522, + "2012211287/train1": 0.5289841828268461, + "2012211287/train2": 0.531948622673026, + "2038594191/train0": 0.5740754232851301, + "2038594191/train1": 0.5667318952152414, + "2038594191/train2": 0.5635629372227472, + "2054899126/train0": 0.5290379603266726, + "2054899126/train1": 0.5248824145830566, + "2054899126/train2": 0.5308674006096046, + "2061635888/train0": 0.5393032283831621, + "2061635888/train1": 0.5362308522692728, + "2061635888/train2": 0.5391540330154272, + "211245842/train0": 0.517501300906042, + "211245842/train1": 0.5160911960797463, + "211245842/train2": 0.5172535124316141, + "278008524/train0": 0.5325709273453999, + "278008524/train1": 0.5322397110343211, + "278008524/train2": 0.536003409987186, + "307068879/train0": 0.5260813668824347, + "307068879/train1": 0.5218750448469349, + "307068879/train2": 0.5232228710238422, + "412102563/train0": 0.5444159724840614, + "412102563/train1": 0.5435121854300012, + "412102563/train2": 0.5454433115811214, + "440577711/train0": 0.5273358125858131, + "440577711/train1": 0.5197720209730798, + "440577711/train2": 0.5240762582462818, + "441046872/train0": 0.5267148410723766, + "441046872/train1": 0.5234185829111347, + "441046872/train2": 0.5255994659745129, + "509430392/train0": 0.5129229068912616, + "509430392/train1": 0.5125583130541626, + "509430392/train2": 0.5151111613972951, + "585904130/train0": 0.5296207411344719, + "585904130/train1": 0.5273196708342051, + "585904130/train2": 0.529609307543077, + "608016290/train0": 0.5561217648468477, + "608016290/train1": 0.5555371517280593, + "608016290/train2": 0.5609759280259827, + "626486813/train0": 0.5656883276900622, + "626486813/train1": 0.5625549213649634, + "626486813/train2": 0.5636531135559604, + "636680137/train0": 0.5190311348285646, + "636680137/train1": 0.5160986674159838, + "636680137/train2": 0.5129089770472632, + "655884605/train0": 0.5597351518736766, + "655884605/train1": 0.562343415271088, + "655884605/train2": 0.5588372328026256, + "672328262/train0": 0.5525242267379454, + "672328262/train1": 0.5406287558110948, + "672328262/train2": 0.5442903896362881, + "701667486/train0": 0.5307788011897017, + "701667486/train1": 0.5325935443428202, + "701667486/train2": 0.5303820261796057, + "747260437/train0": 0.5222884856095076, + "747260437/train1": 0.5252521749807111, + "747260437/train2": 0.5187728818008999, + "750448296/train0": 0.5207629106352885, + "750448296/train1": 0.5196740117740427, + "750448296/train2": 0.5222504044176294, + "767747424/train0": 0.5286089316154619, + "767747424/train1": 0.5262713091494071, + "767747424/train2": 0.5306118103750065 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5379868829976331, + "low": 0.5233696556864914, + "mean": 0.5315865513366932, + "n": 5, + "num_samples": 300 + }, + "mean": 0.5315865513366932, + "per_task": { + "LiftPegUpright-v1": 0.5321483671612611, + "PickCube-v1": 0.5397879799473912, + "PullCube-v1": 0.5315713773974821, + "PushCube-v1": 0.5154259344288218, + "StackCube-v1": 0.5389990977485094 + } + }, + "metric": "pairwise_causal_calibration_confidence", + "micro": { + "confidence": 0.95, + "high": 0.5347090519983646, + "low": 0.5285313883446271, + "mean": 0.5314829821239634, + "n": 138, + "num_samples": 300 + }, + "n": 138 + }, + "pairwise_causal_calibration_ece": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.12306181225731887, + "low": 0.09531198783107489, + "mean": 0.10993892641652644, + "n": 120, + "num_samples": 300 + }, + "mean": 0.10993892641652644, + "per_seed": { + "1073161778/train0": 0.09663644065605714, + "1073161778/train1": 0.08553249499650317, + "1073161778/train2": 0.14116750640137155, + "1113371261/train0": 0.10961166265259512, + "1113371261/train1": 0.06848091739835714, + "1113371261/train2": 0.036081505594097285, + "1151410346/train0": 0.015348396032020872, + "1151410346/train1": 0.08122051010327141, + "1151410346/train2": 0.14951101187259608, + "1261930456/train0": 0.06290376649689283, + "1261930456/train1": 0.007916493303327332, + "1261930456/train2": 0.0998231318859727, + "1324133122/train0": 0.16288330730273656, + "1324133122/train1": 0.10154169690861004, + "1324133122/train2": 0.33862520696123877, + "1324642411/train0": 0.04975865420062986, + "1324642411/train1": 0.1727565109927809, + "1324642411/train2": 0.010384144566828457, + "1375492786/train0": 0.1376918032206741, + "1375492786/train1": 0.03564050994268986, + "1375492786/train2": 0.27581542526587133, + "141984615/train0": 0.10656362935518632, + "141984615/train1": 0.049021298483554854, + "141984615/train2": 0.007445709717532223, + "1461069713/train0": 0.20617662584047913, + "1461069713/train1": 0.09747818257770119, + "1461069713/train2": 0.08149060764001503, + "1480856810/train0": 0.3115896861690668, + "1480856810/train1": 0.061820560810436964, + "1480856810/train2": 0.045050263785638145, + "1509980664/train0": 0.10547718914222602, + "1509980664/train1": 0.29909257573247366, + "1509980664/train2": 0.05022629350677536, + "1522272686/train0": 0.03584865687586758, + "1522272686/train1": 0.32013412762304877, + "1522272686/train2": 0.12449226645436867, + "1549324805/train0": 0.15494101616573086, + "1549324805/train1": 0.06874616583843707, + "1549324805/train2": 0.06550710080767463, + "1578990937/train0": 0.15007327439686133, + "1578990937/train1": 0.11754325880699122, + "1578990937/train2": 0.18468341785799877, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.12595877595550709, + "1790188787/train1": 0.1295014384443925, + "1790188787/train2": 0.12650226276678844, + "1822210471/train0": 0.08571587301053908, + "1822210471/train1": 0.09545626012360084, + "1822210471/train2": 0.12372642714412677, + "1845489013/train0": 0.18168602419877433, + "1845489013/train1": 0.4790352714246695, + "1845489013/train2": 0.07280554245739407, + "1999022574/train0": 0.2787192635458735, + "1999022574/train1": 0.041669823510756876, + "1999022574/train2": 0.10572081472129435, + "2012211287/train0": 0.1332754039717749, + "2012211287/train1": 0.04601581717315384, + "2012211287/train2": 0.1528339860226262, + "2038594191/train0": 0.2982133543196128, + "2038594191/train1": 0.27043559891894486, + "2038594191/train2": 0.1090174826772927, + "2054899126/train0": 0.13118564683000297, + "2054899126/train1": 0.06212019621724363, + "2054899126/train2": 0.024482266081752357, + "2061635888/train0": 0.06094260950628667, + "2061635888/train1": 0.06400863004705062, + "2061635888/train2": 0.016401522540128385, + "211245842/train0": 0.07073399321160512, + "211245842/train1": 0.14224213725358703, + "211245842/train2": 0.04941315423505255, + "278008524/train0": 0.12646220200877553, + "278008524/train1": 0.10215544676823522, + "278008524/train2": 0.10720913222134038, + "307068879/train0": 0.036558379935610945, + "307068879/train1": 0.049553526581636476, + "307068879/train2": 0.028717376518347715, + "412102563/train0": 0.04441597248406137, + "412102563/train1": 0.22571858380076806, + "412102563/train2": 0.1644909306287403, + "440577711/train0": 0.09335384258660073, + "440577711/train1": 0.11977202097307982, + "440577711/train2": 0.061113295283318825, + "441046872/train0": 0.163078477436013, + "441046872/train1": 0.16749050799795617, + "441046872/train2": 0.07105401142905837, + "509430392/train0": 0.10052247125999891, + "509430392/train1": 0.049346448850599356, + "509430392/train2": 0.0815274940648898, + "585904130/train0": 0.06533502684875758, + "585904130/train1": 0.07268032916579492, + "585904130/train2": 0.25220887427510486, + "608016290/train0": 0.03209834914225741, + "608016290/train1": 0.06029905648996403, + "608016290/train2": 0.05264259469264974, + "626486813/train0": 0.1169840028584393, + "626486813/train1": 0.19272229094655288, + "626486813/train2": 0.10846025594242703, + "636680137/train0": 0.10236446816189787, + "636680137/train1": 0.1839013325840162, + "636680137/train2": 0.012908977047263237, + "655884605/train0": 0.05616248877449882, + "655884605/train1": 0.11789897082664356, + "655884605/train2": 0.027075654701186606, + "672328262/train0": 0.27100518502676063, + "672328262/train1": 0.08348589866823782, + "672328262/train2": 0.1442903896362882, + "701667486/train0": 0.00696927738017783, + "701667486/train1": 0.06264455089527499, + "701667486/train2": 0.011863507661087236, + "747260437/train0": 0.03062181894284094, + "747260437/train1": 0.15808115835262226, + "747260437/train2": 0.08122711819910011, + "750448296/train0": 0.08615947226034647, + "750448296/train1": 0.13755775384340305, + "750448296/train2": 0.10634938783489285, + "767747424/train0": 0.1306918529485389, + "767747424/train1": 0.13822538375258459, + "767747424/train2": 0.13666304064151388 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.13873127819057124, + "low": 0.0957185382719192, + "mean": 0.1139495708835625, + "n": 5, + "num_samples": 300 + }, + "mean": 0.1139495708835625, + "per_task": { + "LiftPegUpright-v1": 0.08779692862966537, + "PickCube-v1": 0.16015835031776168, + "PullCube-v1": 0.1202240575596694, + "PushCube-v1": 0.09497784791093052, + "StackCube-v1": 0.10659066999978553 + } + }, + "metric": "pairwise_causal_calibration_ece", + "micro": { + "confidence": 0.95, + "high": 0.12486695399157724, + "low": 0.09782132803662658, + "mean": 0.1113981018733552, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "pairwise_causal_calibration_pairs": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 88.32847222222222, + "low": 75.17222222222223, + "mean": 82.21180555555556, + "n": 120, + "num_samples": 300 + }, + "mean": 82.21180555555556, + "per_seed": { + "1073161778/train0": 119.0, + "1073161778/train1": 120.0, + "1073161778/train2": 84.0, + "1113371261/train0": 28.0, + "1113371261/train1": 28.0, + "1113371261/train2": 28.0, + "1151410346/train0": 105.0, + "1151410346/train1": 105.0, + "1151410346/train2": 105.0, + "1261930456/train0": 99.0, + "1261930456/train1": 110.0, + "1261930456/train2": 84.0, + "1324133122/train0": 28.0, + "1324133122/train1": 28.0, + "1324133122/train2": 28.0, + "1324642411/train0": 120.0, + "1324642411/train1": 119.0, + "1324642411/train2": 110.0, + "1375492786/train0": 75.0, + "1375492786/train1": 114.0, + "1375492786/train2": 84.0, + "141984615/train0": 120.0, + "141984615/train1": 120.0, + "141984615/train2": 120.0, + "1461069713/train0": 120.0, + "1461069713/train1": 120.0, + "1461069713/train2": 120.0, + "1480856810/train0": 99.0, + "1480856810/train1": 119.0, + "1480856810/train2": 119.0, + "1509980664/train0": 90.0, + "1509980664/train1": 63.0, + "1509980664/train2": 90.0, + "1522272686/train0": 36.0, + "1522272686/train1": 36.0, + "1522272686/train2": 36.0, + "1549324805/train0": 120.0, + "1549324805/train1": 120.0, + "1549324805/train2": 120.0, + "1578990937/train0": 117.0, + "1578990937/train1": 110.0, + "1578990937/train2": 119.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 110.0, + "1790188787/train1": 114.0, + "1790188787/train2": 110.0, + "1822210471/train0": 40.0, + "1822210471/train1": 49.0, + "1822210471/train2": 40.0, + "1845489013/train0": 46.0, + "1845489013/train1": 13.0, + "1845489013/train2": 25.0, + "1999022574/train0": 70.0, + "1999022574/train1": 70.0, + "1999022574/train2": 76.0, + "2012211287/train0": 119.0, + "2012211287/train1": 120.0, + "2012211287/train2": 92.0, + "2038594191/train0": 29.0, + "2038594191/train1": 54.0, + "2038594191/train2": 99.0, + "2054899126/train0": 60.0, + "2054899126/train1": 57.0, + "2054899126/train2": 59.5, + "2061635888/train0": 36.0, + "2061635888/train1": 36.0, + "2061635888/train2": 36.0, + "211245842/train0": 119.0, + "211245842/train1": 120.0, + "211245842/train2": 120.0, + "278008524/train0": 109.5, + "278008524/train1": 106.0, + "278008524/train2": 109.5, + "307068879/train0": 91.0, + "307068879/train1": 91.0, + "307068879/train2": 91.0, + "412102563/train0": 60.0, + "412102563/train1": 39.0, + "412102563/train2": 84.0, + "440577711/train0": 29.0, + "440577711/train1": 15.0, + "440577711/train2": 54.0, + "441046872/train0": 55.0, + "441046872/train1": 55.0, + "441046872/train2": 55.0, + "509430392/train0": 119.0, + "509430392/train1": 105.0, + "509430392/train2": 119.0, + "585904130/train0": 84.0, + "585904130/train1": 75.0, + "585904130/train2": 110.0, + "608016290/train0": 105.0, + "608016290/train1": 105.0, + "608016290/train2": 120.0, + "626486813/train0": 120.0, + "626486813/train1": 120.0, + "626486813/train2": 120.0, + "636680137/train0": 120.0, + "636680137/train1": 120.0, + "636680137/train2": 120.0, + "655884605/train0": 36.0, + "655884605/train1": 36.0, + "655884605/train2": 36.0, + "672328262/train0": 17.0, + "672328262/train1": 35.0, + "672328262/train2": 30.0, + "701667486/train0": 42.0, + "701667486/train1": 84.0, + "701667486/train2": 54.0, + "747260437/train0": 120.0, + "747260437/train1": 120.0, + "747260437/train2": 120.0, + "750448296/train0": 108.0, + "750448296/train1": 101.66666666666667, + "750448296/train2": 108.0, + "767747424/train0": 119.25, + "767747424/train1": 119.0, + "767747424/train2": 119.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 95.32888888888888, + "low": 54.05090909090909, + "mean": 77.60379797979797, + "n": 5, + "num_samples": 300 + }, + "mean": 77.60379797979797, + "per_task": { + "LiftPegUpright-v1": 94.8, + "PickCube-v1": 36.0, + "PullCube-v1": 92.32, + "PushCube-v1": 97.44444444444444, + "StackCube-v1": 67.45454545454545 + } + }, + "metric": "pairwise_causal_calibration_pairs", + "micro": { + "confidence": 0.95, + "high": 92.36111111111111, + "low": 80.19444444444444, + "mean": 86.34722222222223, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.12714647100500526, + "low": 0.045656196216331324, + "mean": 0.08627731575280823, + "n": 120, + "num_samples": 300 + }, + "mean": 0.08627731575280823, + "per_seed": { + "1073161778/train0": 0.07106166187087547, + "1073161778/train1": -0.05584757187568534, + "1073161778/train2": 0.07873777798305576, + "1113371261/train0": -0.1171582058773368, + "1113371261/train1": -0.11952381111596475, + "1113371261/train2": -0.2072636199490521, + "1151410346/train0": 0.12464332364425626, + "1151410346/train1": 0.10435665617161227, + "1151410346/train2": 0.0965746708031926, + "1261930456/train0": -0.08116638752440412, + "1261930456/train1": -0.09547952648462638, + "1261930456/train2": -0.0006632054109868557, + "1324133122/train0": -0.09117531897647907, + "1324133122/train1": 0.10108263691246488, + "1324133122/train2": -0.05985917108029659, + "1324642411/train0": 0.5936226389105781, + "1324642411/train1": 0.3040303172517523, + "1324642411/train2": 0.07036169250445845, + "1375492786/train0": -0.29978209413001944, + "1375492786/train1": -0.79693146060142, + "1375492786/train2": -0.6377766658396278, + "141984615/train0": 0.08363360102690726, + "141984615/train1": 0.1560213376803143, + "141984615/train2": -0.046015598870960614, + "1461069713/train0": 0.09082588792911983, + "1461069713/train1": 0.1449663100493438, + "1461069713/train2": 0.13654486820462297, + "1480856810/train0": 0.8247860886762404, + "1480856810/train1": 0.5725138287112228, + "1480856810/train2": 0.4285300550683504, + "1509980664/train0": -0.11725750852840032, + "1509980664/train1": -0.10799597937369632, + "1509980664/train2": -0.17204314625253667, + "1522272686/train0": -0.37628277603589594, + "1522272686/train1": -0.18697224010199912, + "1522272686/train2": -0.2402513599376978, + "1549324805/train0": 0.495453729546493, + "1549324805/train1": 0.18733344993267076, + "1549324805/train2": 0.3994404854817541, + "1578990937/train0": 0.2626760081803321, + "1578990937/train1": 0.17364935131576656, + "1578990937/train2": 0.15403622504955924, + "1747178030/train0": 0.04624259356089375, + "1747178030/train1": 0.00797935515478132, + "1747178030/train2": 0.06617060088497922, + "1790188787/train0": 0.1484177313105067, + "1790188787/train1": 0.13132582655221522, + "1790188787/train2": 0.12795113455003831, + "1822210471/train0": -0.0008074765493970848, + "1822210471/train1": -0.0650108915156051, + "1822210471/train2": -0.0006465763829108795, + "1845489013/train0": -0.22001414669968217, + "1845489013/train1": -0.18822734864823312, + "1845489013/train2": -0.1547908021155039, + "1999022574/train0": 0.0012762962711883319, + "1999022574/train1": -0.08747088123670077, + "1999022574/train2": -0.10186397965031571, + "2012211287/train0": 0.1490312481371141, + "2012211287/train1": -0.036105644427698036, + "2012211287/train2": 0.09375170906247592, + "2038594191/train0": 0.19762473284895746, + "2038594191/train1": 0.3209231912001711, + "2038594191/train2": 0.31779873234459755, + "2054899126/train0": 0.15266150175970736, + "2054899126/train1": 0.1715300859769803, + "2054899126/train2": 0.16346432037867953, + "2061635888/train0": 0.12372212373443373, + "2061635888/train1": 0.049366979703384384, + "2061635888/train2": 0.015355053119340732, + "211245842/train0": 0.04424689933210735, + "211245842/train1": 0.13077847063672782, + "211245842/train2": 0.04386283650419778, + "278008524/train0": 0.2219557334710413, + "278008524/train1": 0.2843962133295508, + "278008524/train2": 0.24145145365463994, + "307068879/train0": 0.08783512114247624, + "307068879/train1": -0.13881394667704933, + "307068879/train2": 0.14397102567158404, + "412102563/train0": 0.0942295054093702, + "412102563/train1": 0.029952749201257664, + "412102563/train2": -0.004657759279381279, + "440577711/train0": 0.5465320030087847, + "440577711/train1": 0.629918599045802, + "440577711/train2": 0.6241310075694073, + "441046872/train0": 0.046645784157056724, + "441046872/train1": -0.17358234568930467, + "441046872/train2": -0.022380468606574055, + "509430392/train0": 0.4269940390592024, + "509430392/train1": 0.37833433181770104, + "509430392/train2": 0.282290647599035, + "585904130/train0": 0.10499618981526784, + "585904130/train1": 0.06533471439439086, + "585904130/train2": 0.09915967492706912, + "608016290/train0": 0.5011206164338899, + "608016290/train1": 0.2910589829678913, + "608016290/train2": 0.4536901796255097, + "626486813/train0": 0.5216712480085811, + "626486813/train1": 0.4816967914950128, + "626486813/train2": 0.39041494710515146, + "636680137/train0": 0.4324165282764497, + "636680137/train1": 0.0489088109005007, + "636680137/train2": 0.13337534819821673, + "655884605/train0": 0.07276876911827704, + "655884605/train1": 0.1554209538300596, + "655884605/train2": 0.11907170730895571, + "672328262/train0": -0.087075237393225, + "672328262/train1": -0.23567605508015027, + "672328262/train2": -0.14512362014342817, + "701667486/train0": 0.18126309408722283, + "701667486/train1": 0.15700516615012938, + "701667486/train2": 0.1922641707596195, + "747260437/train0": -0.08999271197759756, + "747260437/train1": -0.26165809689080527, + "747260437/train2": -0.061796090652556146, + "750448296/train0": 0.005890085195897528, + "750448296/train1": -0.19810309176890203, + "750448296/train2": -0.15048924424583973, + "767747424/train0": -0.028685105901352856, + "767747424/train1": -0.013317565754431063, + "767747424/train2": 0.02448240692729431 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.22447325377947683, + "low": -0.13330904884281763, + "mean": 0.05126313589726559, + "n": 5, + "num_samples": 300 + }, + "mean": 0.05126313589726559, + "per_task": { + "LiftPegUpright-v1": 0.09934494198504117, + "PickCube-v1": -0.2678354586918643, + "PullCube-v1": 0.0587216273047277, + "PushCube-v1": 0.32846837901195985, + "StackCube-v1": 0.03761618987646351 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.1248668767428929, + "low": 0.0427827948842033, + "mean": 0.08503163117842487, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.673611111111111, + "low": 0.4958333333333333, + "mean": 0.5847222222222223, + "n": 120, + "num_samples": 300 + }, + "mean": 0.5847222222222223, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7563636363636363, + "low": 0.22303030303030305, + "mean": 0.5074747474747474, + "n": 5, + "num_samples": 300 + }, + "mean": 0.5074747474747474, + "per_task": { + "LiftPegUpright-v1": 0.6, + "PickCube-v1": 0.0, + "PullCube-v1": 0.5333333333333333, + "PushCube-v1": 0.8888888888888888, + "StackCube-v1": 0.5151515151515151 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.6458333333333334, + "low": 0.4861111111111111, + "mean": 0.5694444444444444, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.4087191828565362, + "low": 1.0967581196351806, + "mean": 1.251578581963097, + "n": 120, + "num_samples": 300 + }, + "mean": 1.251578581963097, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.10274861752986908, + "1113371261/train1": 0.18093222379684448, + "1113371261/train2": 0.19527167081832886, + "1151410346/train0": 0.8663521409034729, + "1151410346/train1": 2.0, + "1151410346/train2": 0.5522047281265259, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.054989203810691833, + "1324133122/train1": 0.1324586570262909, + "1324133122/train2": 0.19291165471076965, + "1324642411/train0": 0.14406004548072815, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.015177905559539795, + "141984615/train1": 0.037011127918958664, + "141984615/train2": 0.058951716870069504, + "1461069713/train0": 0.06019836664199829, + "1461069713/train1": 0.016585012897849083, + "1461069713/train2": 0.09997648000717163, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 0.18325337767601013, + "1522272686/train1": 0.07014383375644684, + "1522272686/train2": 0.14937306940555573, + "1549324805/train0": 0.060871422290802, + "1549324805/train1": 0.19892151653766632, + "1549324805/train2": 0.11310184001922607, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.077283151447773, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.09799930453300476, + "2061635888/train1": 0.07580336928367615, + "2061635888/train2": 0.11105132102966309, + "211245842/train0": 2.0, + "211245842/train1": 0.21295717358589172, + "211245842/train2": 0.23388276994228363, + "278008524/train0": 1.0733563229441643, + "278008524/train1": 1.0376745946705341, + "278008524/train2": 1.1359489113092422, + "307068879/train0": 0.2115575522184372, + "307068879/train1": 0.5876073241233826, + "307068879/train2": 0.2302134484052658, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 0.7233588695526123, + "441046872/train2": 0.7191348671913147, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.7117276191711426, + "626486813/train0": 0.04469547560438514, + "626486813/train1": 0.035827874671667814, + "626486813/train2": 0.04489925503730774, + "636680137/train0": 0.08826130628585815, + "636680137/train1": 0.06741118431091309, + "636680137/train2": 0.0756516233086586, + "655884605/train0": 0.13732868432998657, + "655884605/train1": 0.8128758072853088, + "655884605/train2": 0.21447330713272095, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.10839563608169556, + "747260437/train1": 0.06328952312469482, + "747260437/train2": 0.12510187923908234, + "750448296/train0": 0.7526603279014429, + "750448296/train1": 1.3705032741030057, + "750448296/train2": 1.3504486680030823, + "767747424/train0": 1.0424291878007352, + "767747424/train1": 1.0629530912265182, + "767747424/train2": 1.0671670949086547 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.5757816767742236, + "low": 0.5943610860542816, + "mean": 1.1196794969856438, + "n": 5, + "num_samples": 300 + }, + "mean": 1.1196794969856438, + "per_task": { + "LiftPegUpright-v1": 1.4122399787108104, + "PickCube-v1": 0.13425676027933756, + "PullCube-v1": 1.1093241602430741, + "PushCube-v1": 1.7857814149724112, + "StackCube-v1": 1.1567951707225856 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.3550500385204538, + "low": 1.0635484514318199, + "mean": 1.2159999172436073, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 6.555555555555556, + "low": 4.948611111111111, + "mean": 5.677777777777778, + "n": 120, + "num_samples": 300 + }, + "mean": 5.677777777777778, + "per_seed": { + "1073161778/train0": 6.0, + "1073161778/train1": 7.0, + "1073161778/train2": 6.0, + "1113371261/train0": 2.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 9.0, + "1151410346/train1": 3.0, + "1151410346/train2": 8.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 4.0, + "1324133122/train0": 6.0, + "1324133122/train1": 2.0, + "1324133122/train2": 1.0, + "1324642411/train0": 13.0, + "1324642411/train1": 4.0, + "1324642411/train2": 2.0, + "1375492786/train0": 3.0, + "1375492786/train1": 4.0, + "1375492786/train2": 2.0, + "141984615/train0": 6.0, + "141984615/train1": 15.0, + "141984615/train2": 3.0, + "1461069713/train0": 1.0, + "1461069713/train1": 13.0, + "1461069713/train2": 3.0, + "1480856810/train0": 0.0, + "1480856810/train1": 9.0, + "1480856810/train2": 4.0, + "1509980664/train0": 10.0, + "1509980664/train1": 9.0, + "1509980664/train2": 9.0, + "1522272686/train0": 1.0, + "1522272686/train1": 2.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 9.0, + "1578990937/train1": 0.0, + "1578990937/train2": 7.0, + "1747178030/train0": 3.0, + "1747178030/train1": 7.0, + "1747178030/train2": 15.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 9.0, + "1822210471/train0": 5.0, + "1822210471/train1": 3.0, + "1822210471/train2": 4.0, + "1845489013/train0": 5.0, + "1845489013/train1": 1.0, + "1845489013/train2": 8.0, + "1999022574/train0": 5.0, + "1999022574/train1": 8.0, + "1999022574/train2": 7.0, + "2012211287/train0": 13.0, + "2012211287/train1": 13.0, + "2012211287/train2": 14.0, + "2038594191/train0": 5.0, + "2038594191/train1": 9.0, + "2038594191/train2": 3.0, + "2054899126/train0": 4.5, + "2054899126/train1": 7.0, + "2054899126/train2": 8.0, + "2061635888/train0": 3.0, + "2061635888/train1": 2.0, + "2061635888/train2": 7.0, + "211245842/train0": 8.0, + "211245842/train1": 4.0, + "211245842/train2": 4.0, + "278008524/train0": 10.0, + "278008524/train1": 8.0, + "278008524/train2": 14.0, + "307068879/train0": 11.0, + "307068879/train1": 6.0, + "307068879/train2": 3.0, + "412102563/train0": 10.0, + "412102563/train1": 8.0, + "412102563/train2": 7.0, + "440577711/train0": 1.0, + "440577711/train1": 10.0, + "440577711/train2": 13.0, + "441046872/train0": 7.0, + "441046872/train1": 6.0, + "441046872/train2": 1.0, + "509430392/train0": 15.0, + "509430392/train1": 3.0, + "509430392/train2": 1.0, + "585904130/train0": 4.0, + "585904130/train1": 15.0, + "585904130/train2": 1.0, + "608016290/train0": 11.0, + "608016290/train1": 3.0, + "608016290/train2": 1.0, + "626486813/train0": 11.0, + "626486813/train1": 10.0, + "626486813/train2": 2.0, + "636680137/train0": 0.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 0.0, + "655884605/train1": 6.0, + "655884605/train2": 1.0, + "672328262/train0": 5.0, + "672328262/train1": 3.0, + "672328262/train2": 4.0, + "701667486/train0": 10.0, + "701667486/train1": 11.0, + "701667486/train2": 7.0, + "747260437/train0": 14.0, + "747260437/train1": 4.0, + "747260437/train2": 6.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 8.333333333333334, + "750448296/train2": 10.333333333333334, + "767747424/train0": 7.0, + "767747424/train1": 7.0, + "767747424/train2": 5.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 6.673212121212122, + "low": 3.1532121212121216, + "mean": 5.104323232323233, + "n": 5, + "num_samples": 300 + }, + "mean": 5.104323232323233, + "per_task": { + "LiftPegUpright-v1": 7.2, + "PickCube-v1": 1.3333333333333333, + "PullCube-v1": 6.16, + "PushCube-v1": 5.222222222222222, + "StackCube-v1": 5.606060606060606 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 6.548611111111111, + "low": 5.208333333333333, + "mean": 5.923611111111111, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.42916666666666664, + "low": 0.2604166666666667, + "mean": 0.33958333333333335, + "n": 120, + "num_samples": 300 + }, + "mean": 0.33958333333333335, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.41393939393939394, + "low": 0.12460606060606061, + "mean": 0.27793939393939393, + "n": 5, + "num_samples": 300 + }, + "mean": 0.27793939393939393, + "per_task": { + "LiftPegUpright-v1": 0.26666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.32, + "PushCube-v1": 0.5, + "StackCube-v1": 0.30303030303030304 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.3888888888888889, + "low": 0.2569444444444444, + "mean": 0.3263888888888889, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.1, + "low": -0.029166666666666667, + "mean": 0.03333333333333333, + "n": 120, + "num_samples": 300 + }, + "mean": 0.03333333333333333, + "per_seed": { + "1073161778/train0": -1.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": -1.0, + "2038594191/train1": -1.0, + "2038594191/train2": -1.0, + "2054899126/train0": -0.5, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": -1.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": -1.0, + "747260437/train1": 0.0, + "747260437/train2": -1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.21333333333333332, + "low": -0.010666666666666668, + "mean": 0.07806060606060605, + "n": 5, + "num_samples": 300 + }, + "mean": 0.07806060606060605, + "per_task": { + "LiftPegUpright-v1": 0.06666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": -0.04, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.030303030303030304 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.09722222222222222, + "low": -0.034722222222222224, + "mean": 0.034722222222222224, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9767636695663291, + "low": 0.6694472551869696, + "mean": 0.8134982612139235, + "n": 120, + "num_samples": 300 + }, + "mean": 0.8134982612139235, + "per_seed": { + "1073161778/train0": 0.67014479637146, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.11045536398887634, + "1324642411/train1": 0.11658626794815063, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.007176061626523733, + "141984615/train1": 0.030119141563773155, + "141984615/train2": 0.02716982364654541, + "1461069713/train0": 0.014822185039520264, + "1461069713/train1": 0.006729066371917725, + "1461069713/train2": 0.09997648000717163, + "1480856810/train0": 2.0, + "1480856810/train1": 0.21981161832809448, + "1480856810/train2": 0.21894000470638275, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.0260902252048254, + "1549324805/train1": 0.0303483996540308, + "1549324805/train2": 0.04690708592534065, + "1578990937/train0": 2.0, + "1578990937/train1": 0.12732218205928802, + "1578990937/train2": 0.15726424753665924, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 0.10585354268550873, + "1790188787/train2": 0.10904359817504883, + "1822210471/train0": 0.06247224286198616, + "1822210471/train1": 0.045472364872694016, + "1822210471/train2": 0.08447941392660141, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7026570439338684, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.6934840679168701, + "2038594191/train0": 0.04285776615142822, + "2038594191/train1": 0.07674666494131088, + "2038594191/train2": 0.12977690994739532, + "2054899126/train0": 1.0403175354003906, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 1.0287093725055456, + "278008524/train1": 1.0095977680757642, + "278008524/train2": 1.0227726306766272, + "307068879/train0": 0.19761089980602264, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.22035211324691772, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.20773810148239136, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 0.11310504376888275, + "509430392/train1": 2.0, + "509430392/train2": 2.682209014892578e-07, + "585904130/train0": 0.09919407218694687, + "585904130/train1": 0.04792279005050659, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.6789124011993408, + "626486813/train0": 0.009900043485686183, + "626486813/train1": 0.006450424436479807, + "626486813/train2": 0.01727360486984253, + "636680137/train0": 0.056092143058776855, + "636680137/train1": 0.017789265140891075, + "636680137/train2": 0.015303333289921284, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.11176733672618866, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 0.04013325646519661, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.061248183250427246, + "747260437/train1": 2.0, + "747260437/train2": 0.06438694149255753, + "750448296/train0": 0.125017994393905, + "750448296/train1": 0.109618179500103, + "750448296/train2": 0.09164678926269214, + "767747424/train0": 0.5305264240596443, + "767747424/train1": 0.5966028785333037, + "767747424/train2": 0.5695318393409252 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9610522832274435, + "low": 0.6580356173443072, + "mean": 0.8072238209012783, + "n": 5, + "num_samples": 300 + }, + "mean": 0.8072238209012783, + "per_task": { + "LiftPegUpright-v1": 0.9396337101856868, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.6822500004371007, + "PushCube-v1": 1.0611259018381436, + "StackCube-v1": 0.769150389414845 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.9093973265650371, + "low": 0.6554252143218441, + "mean": 0.7742872366474735, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.14305571060378697, + "low": -0.09107819030889205, + "mean": 0.023380198916129302, + "n": 120, + "num_samples": 300 + }, + "mean": 0.023380198916129302, + "per_seed": { + "1073161778/train0": -1.32985520362854, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0039848387241363525, + "1324642411/train1": 0.010115742683410645, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": -0.09565818542614579, + "141984615/train1": -0.07271510548889637, + "141984615/train2": -0.07566442340612411, + "1461069713/train0": -0.07908087968826294, + "1461069713/train1": -0.08717399835586548, + "1461069713/train2": 0.006073415279388428, + "1480856810/train0": 1.9129659235477448, + "1480856810/train1": 0.13277754187583923, + "1480856810/train2": 0.1319059282541275, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": -0.07842461206018925, + "1549324805/train1": -0.07416643761098385, + "1549324805/train2": -0.057607751339673996, + "1578990937/train0": 1.17586350440979, + "1578990937/train1": -0.6968143135309219, + "1578990937/train2": -0.6668722480535507, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.87523253262043, + "1790188787/train1": -0.01891392469406128, + "1790188787/train2": -0.01572386920452118, + "1822210471/train0": -0.009443063288927078, + "1822210471/train1": -0.026442941278219223, + "1822210471/train2": 0.012564107775688171, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": -0.020948290824890137, + "2012211287/train1": 0.0, + "2012211287/train2": -0.030121266841888428, + "2038594191/train0": -1.9571422338485718, + "2038594191/train1": -1.9232533350586891, + "2038594191/train2": -1.8702230900526047, + "2054899126/train0": -0.9596824645996094, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.985974382609129, + "278008524/train1": 0.9668627781793475, + "278008524/train2": 0.9800376407802105, + "307068879/train0": -0.4024796634912491, + "307068879/train1": 0.0, + "307068879/train2": -0.379738450050354, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.9638472199440002, + "440577711/train1": 1.9638472199440002, + "440577711/train2": 1.9638472199440002, + "441046872/train0": -0.4922170639038086, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.11310482025146484, + "509430392/train1": 1.999999776482582, + "509430392/train2": 4.470348358154297e-08, + "585904130/train0": -0.0825214758515358, + "585904130/train1": -0.13379275798797607, + "585904130/train2": 1.8182844519615173, + "608016290/train0": 1.274298369884491, + "608016290/train1": 1.274298369884491, + "608016290/train2": -0.04678922891616821, + "626486813/train0": 0.004801700764801353, + "626486813/train1": 0.0013520817155949771, + "626486813/train2": 0.0121752621489577, + "636680137/train0": -0.027042903006076813, + "636680137/train1": -0.0653457809239626, + "636680137/train2": -0.06783171277493238, + "655884605/train0": 0.0, + "655884605/train1": -0.051239341497421265, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": -1.9598667435348034, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": -1.9387518167495728, + "747260437/train1": 0.0, + "747260437/train2": -1.9356130585074425, + "750448296/train0": 0.007424845049778621, + "750448296/train1": -0.007974969844023386, + "750448296/train2": -0.02594636008143425, + "767747424/train0": -0.04267291794531047, + "767747424/train1": 0.023403536528348923, + "767747424/train2": -0.0036675026640295982 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.39773680510846043, + "low": -0.05779964217698822, + "mean": 0.11166402372726322, + "n": 5, + "num_samples": 300 + }, + "mean": 0.11166402372726322, + "per_task": { + "LiftPegUpright-v1": -0.010236828525861105, + "PickCube-v1": 0.0, + "PullCube-v1": -0.09292046078636, + "PushCube-v1": 0.6687217230598131, + "StackCube-v1": -0.007244315111275875 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.15572734094651727, + "low": -0.09504848822366006, + "mean": 0.03246765020513623, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5917341341859557, + "low": 0.35403546871255254, + "mean": 0.4665542311901744, + "n": 120, + "num_samples": 300 + }, + "mean": 0.4665542311901744, + "per_seed": { + "1073161778/train0": 1.32985520362854, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.02250880002975464, + "1113371261/train2": 0.036848247051239014, + "1151410346/train0": 0.6403508484363556, + "1151410346/train1": 1.7739987075328827, + "1151410346/train2": 0.32620343565940857, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.04025374352931976, + "1324133122/train2": 0.10070674121379852, + "1324642411/train0": 0.03360468149185181, + "1324642411/train1": 1.8834137320518494, + "1324642411/train2": 1.89352947473526, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.008001843933016062, + "141984615/train1": 0.006891986355185509, + "141984615/train2": 0.031781893223524094, + "1461069713/train0": 0.04537618160247803, + "1461069713/train1": 0.009855946525931358, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.7801883816719055, + "1480856810/train2": 1.7810599952936172, + "1509980664/train0": 1.1665899753570557, + "1509980664/train1": 1.1665899753570557, + "1509980664/train2": 1.1665899753570557, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0347811970859766, + "1549324805/train1": 0.16857311688363552, + "1549324805/train2": 0.06619475409388542, + "1578990937/train0": 0.0, + "1578990937/train1": 1.872677817940712, + "1578990937/train2": 1.8427357524633408, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.8941464573144913, + "1790188787/train2": 1.8909564018249512, + "1822210471/train0": 1.9375277571380138, + "1822210471/train1": 1.954527635127306, + "1822210471/train2": 1.9155205860733986, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.2973429560661316, + "2012211287/train1": 1.2763946652412415, + "2012211287/train2": 1.3065159320831299, + "2038594191/train0": 1.9571422338485718, + "2038594191/train1": 1.9232533350586891, + "2038594191/train2": 1.8702230900526047, + "2054899126/train0": 0.036965616047382355, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.04574880003929138, + "2061635888/train1": 0.02355286478996277, + "2061635888/train2": 0.05880081653594971, + "211245842/train0": 1.8090915828943253, + "211245842/train1": 0.02204875648021698, + "211245842/train2": 0.04297435283660889, + "278008524/train0": 0.04464695043861866, + "278008524/train1": 0.028076826594769955, + "278008524/train2": 0.11317628063261509, + "307068879/train0": 0.01394665241241455, + "307068879/train1": 0.0, + "307068879/train2": 0.009861335158348083, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.7922618985176086, + "441046872/train1": 0.023403704166412354, + "441046872/train2": 0.019179701805114746, + "509430392/train0": 1.8868949562311172, + "509430392/train1": 0.0, + "509430392/train2": 5.155801773071289e-06, + "585904130/train0": 1.9008059278130531, + "585904130/train1": 1.9520772099494934, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.03281521797180176, + "626486813/train0": 0.034795432118698955, + "626486813/train1": 0.029377450235188007, + "626486813/train2": 0.02762565016746521, + "636680137/train0": 0.0321691632270813, + "636680137/train1": 0.04962191917002201, + "636680137/train2": 0.060348290018737316, + "655884605/train0": 0.0, + "655884605/train1": 0.7011084705591202, + "655884605/train2": 0.05146662890911102, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 1.9598667435348034, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.04714745283126831, + "747260437/train1": 0.0, + "747260437/train2": 0.06071493774652481, + "750448296/train0": 0.6276423335075378, + "750448296/train1": 1.2608850946029027, + "750448296/train2": 1.25880187874039, + "767747424/train0": 0.5119027637410909, + "767747424/train1": 0.4663502126932144, + "767747424/train2": 0.4976352555677295 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6077285580653132, + "low": 0.17293525018475273, + "mean": 0.40844574604446837, + "n": 5, + "num_samples": 300 + }, + "mean": 0.40844574604446837, + "per_task": { + "LiftPegUpright-v1": 0.47343848447004955, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4528969661643108, + "PushCube-v1": 0.7246555131342676, + "StackCube-v1": 0.3912377664537141 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.6050060989574477, + "low": 0.3520631702493928, + "mean": 0.4654409392969683, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3256944444444444, + "low": 0.18680555555555556, + "mean": 0.2534722222222222, + "n": 120, + "num_samples": 300 + }, + "mean": 0.2534722222222222, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 1.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3424242424242424, + "low": 0.10909090909090909, + "mean": 0.23220202020202022, + "n": 5, + "num_samples": 300 + }, + "mean": 0.23220202020202022, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.22666666666666666, + "PushCube-v1": 0.3888888888888889, + "StackCube-v1": 0.21212121212121213 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.3263888888888889, + "low": 0.18055555555555555, + "mean": 0.25, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3055555555555555, + "low": 0.15833333333333333, + "mean": 0.23194444444444443, + "n": 120, + "num_samples": 300 + }, + "mean": 0.23194444444444443, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 0.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7023030303030303, + "low": 0.10133333333333334, + "mean": 0.40452525252525257, + "n": 5, + "num_samples": 300 + }, + "mean": 0.40452525252525257, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 1.0, + "PullCube-v1": 0.02666666666666667, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": 0.48484848484848486 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.2708333333333333, + "low": 0.14583333333333334, + "mean": 0.2013888888888889, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5403802280805798, + "low": 0.2915130193099483, + "mean": 0.4232947583840643, + "n": 120, + "num_samples": 300 + }, + "mean": 0.4232947583840643, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.897251382470131, + "1113371261/train1": 1.8190677762031555, + "1113371261/train2": 1.8047283291816711, + "1151410346/train0": 1.133647859096527, + "1151410346/train1": 0.0, + "1151410346/train2": 1.4477952718734741, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.9450107961893082, + "1324133122/train1": 1.867541342973709, + "1324133122/train2": 1.8070883452892303, + "1324642411/train0": 1.8559399545192719, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.270599365234375, + "141984615/train1": 0.24876614287495613, + "141984615/train2": 0.2268255539238453, + "1461069713/train0": 0.0991440862417221, + "1461069713/train1": 0.14275743998587132, + "1461069713/train2": 0.05936597287654877, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.8167466223239899, + "1522272686/train1": 1.9298561662435532, + "1522272686/train2": 1.8506269305944443, + "1549324805/train0": 0.23658466339111328, + "1549324805/train1": 0.09853456914424896, + "1549324805/train2": 0.1843542456626892, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.922716848552227, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.9020006954669952, + "2061635888/train1": 1.9241966307163239, + "2061635888/train2": 1.888948678970337, + "211245842/train0": 0.0, + "211245842/train1": 1.7870428264141083, + "211245842/train2": 1.7661172300577164, + "278008524/train0": 0.04055231809616089, + "278008524/train1": 0.07623404636979103, + "278008524/train2": 0.0, + "307068879/train0": 1.7884424477815628, + "307068879/train1": 1.4123926758766174, + "307068879/train2": 1.7697865515947342, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.2766411304473877, + "441046872/train2": 1.2808651328086853, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.9999945759773254, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.2882723808288574, + "626486813/train0": 0.12161866808310151, + "626486813/train1": 0.12975450651720166, + "626486813/train2": 0.12068312615156174, + "636680137/train0": 0.0823030173778534, + "636680137/train1": 0.10315313935279846, + "636680137/train2": 0.09491270035505295, + "655884605/train0": 1.8626713156700134, + "655884605/train1": 1.1871241927146912, + "655884605/train2": 1.785526692867279, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.17026761174201965, + "747260437/train1": 0.21537372469902039, + "747260437/train2": 0.15356136858463287, + "750448296/train0": 0.6553361676633358, + "750448296/train1": 0.03749322146177292, + "750448296/train2": 0.05754782756169637, + "767747424/train0": 0.06562284985557199, + "767747424/train1": 0.045098946429789066, + "767747424/train2": 0.04088494274765253 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.3095993929816916, + "low": 0.20360193743308388, + "mean": 0.7236978183567826, + "n": 5, + "num_samples": 300 + }, + "mean": 0.7236978183567826, + "per_task": { + "LiftPegUpright-v1": 0.5877600212891897, + "PickCube-v1": 1.8657432397206624, + "PullCube-v1": 0.1075624164690574, + "PushCube-v1": 0.21421858502758873, + "StackCube-v1": 0.8432048292774142 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 0.49125471889015493, + "low": 0.27718254707158646, + "mean": 0.3761285081272945, + "n": 144, + "num_samples": 300 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.006988199735446857, + "low": -0.04978091939463336, + "mean": -0.02190632784962469, + "n": 120, + "num_samples": 300 + }, + "mean": -0.02190632784962469, + "per_seed": { + "1073161778/train0": 0.028235521236802175, + "1073161778/train1": 0.028235521236802175, + "1073161778/train2": 0.028235521236802175, + "1113371261/train0": 0.13235082030966006, + "1113371261/train1": 0.13235082030966006, + "1113371261/train2": 0.13235082030966006, + "1151410346/train0": 0.13235082030966006, + "1151410346/train1": 0.13235082030966006, + "1151410346/train2": 0.13235082030966006, + "1261930456/train0": 0.07420081675594419, + "1261930456/train1": 0.07420081675594419, + "1261930456/train2": 0.07420081675594419, + "1324133122/train0": 0.13235082030966006, + "1324133122/train1": 0.13235082030966006, + "1324133122/train2": 0.13235082030966006, + "1324642411/train0": 0.07420081675594419, + "1324642411/train1": 0.07420081675594419, + "1324642411/train2": 0.07420081675594419, + "1375492786/train0": -0.2012384152043573, + "1375492786/train1": -0.2012384152043573, + "1375492786/train2": -0.2012384152043573, + "141984615/train0": -0.2012384152043573, + "141984615/train1": -0.2012384152043573, + "141984615/train2": -0.2012384152043573, + "1461069713/train0": -0.2012384152043573, + "1461069713/train1": -0.2012384152043573, + "1461069713/train2": -0.2012384152043573, + "1480856810/train0": 0.07420081675594419, + "1480856810/train1": 0.07420081675594419, + "1480856810/train2": 0.07420081675594419, + "1509980664/train0": 0.13235082030966006, + "1509980664/train1": 0.13235082030966006, + "1509980664/train2": 0.13235082030966006, + "1522272686/train0": 0.5025584143631503, + "1522272686/train1": 0.5025584143631503, + "1522272686/train2": 0.5025584143631503, + "1549324805/train0": -0.2012384152043573, + "1549324805/train1": -0.2012384152043573, + "1549324805/train2": -0.2012384152043573, + "1578990937/train0": 0.13235082030966006, + "1578990937/train1": 0.13235082030966006, + "1578990937/train2": 0.13235082030966006, + "1747178030/train0": -0.2012384152043573, + "1747178030/train1": -0.2012384152043573, + "1747178030/train2": -0.2012384152043573, + "1790188787/train0": 0.07420081675594419, + "1790188787/train1": 0.07420081675594419, + "1790188787/train2": 0.07420081675594419, + "1822210471/train0": -0.2012384152043573, + "1822210471/train1": -0.2012384152043573, + "1822210471/train2": -0.2012384152043573, + "1845489013/train0": -0.2012384152043573, + "1845489013/train1": -0.2012384152043573, + "1845489013/train2": -0.2012384152043573, + "1999022574/train0": 0.13235082030966006, + "1999022574/train1": 0.13235082030966006, + "1999022574/train2": 0.13235082030966006, + "2012211287/train0": 0.028235521236802175, + "2012211287/train1": 0.028235521236802175, + "2012211287/train2": 0.028235521236802175, + "2038594191/train0": -0.2012384152043573, + "2038594191/train1": -0.2012384152043573, + "2038594191/train2": -0.2012384152043573, + "2054899126/train0": -0.2012384152043573, + "2054899126/train1": -0.2012384152043573, + "2054899126/train2": -0.2012384152043573, + "2061635888/train0": 0.13235082030966006, + "2061635888/train1": 0.13235082030966006, + "2061635888/train2": 0.13235082030966006, + "211245842/train0": 0.13235082030966006, + "211245842/train1": 0.13235082030966006, + "211245842/train2": 0.13235082030966006, + "278008524/train0": -0.2012384152043573, + "278008524/train1": -0.2012384152043573, + "278008524/train2": -0.2012384152043573, + "307068879/train0": 0.028235521236802175, + "307068879/train1": 0.028235521236802175, + "307068879/train2": 0.028235521236802175, + "412102563/train0": 0.13235082030966006, + "412102563/train1": 0.13235082030966006, + "412102563/train2": 0.13235082030966006, + "440577711/train0": 0.07420081675594419, + "440577711/train1": 0.07420081675594419, + "440577711/train2": 0.07420081675594419, + "441046872/train0": 0.028235521236802175, + "441046872/train1": 0.028235521236802175, + "441046872/train2": 0.028235521236802175, + "509430392/train0": 0.07420081675594419, + "509430392/train1": 0.07420081675594419, + "509430392/train2": 0.07420081675594419, + "585904130/train0": -0.2012384152043573, + "585904130/train1": -0.2012384152043573, + "585904130/train2": -0.2012384152043573, + "608016290/train0": 0.028235521236802175, + "608016290/train1": 0.028235521236802175, + "608016290/train2": 0.028235521236802175, + "626486813/train0": -0.2012384152043573, + "626486813/train1": -0.2012384152043573, + "626486813/train2": -0.2012384152043573, + "636680137/train0": -0.2012384152043573, + "636680137/train1": -0.2012384152043573, + "636680137/train2": -0.2012384152043573, + "655884605/train0": 0.13235082030966006, + "655884605/train1": 0.13235082030966006, + "655884605/train2": 0.13235082030966006, + "672328262/train0": -0.2012384152043573, + "672328262/train1": -0.2012384152043573, + "672328262/train2": -0.2012384152043573, + "701667486/train0": 0.13235082030966006, + "701667486/train1": 0.13235082030966006, + "701667486/train2": 0.13235082030966006, + "747260437/train0": -0.2012384152043573, + "747260437/train1": -0.2012384152043573, + "747260437/train2": -0.2012384152043573, + "750448296/train0": -0.20123841520435728, + "750448296/train1": -0.20123841520435728, + "750448296/train2": -0.20123841520435728, + "767747424/train0": -0.2012384152043573, + "767747424/train1": -0.2012384152043573, + "767747424/train2": -0.2012384152043573 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.32202231621643945, + "low": -0.08862578081332213, + "mean": 0.10722143149223981, + "n": 5, + "num_samples": 300 + }, + "mean": 0.10722143149223981, + "per_task": { + "LiftPegUpright-v1": 0.028235521236802175, + "PickCube-v1": 0.5025584143631503, + "PullCube-v1": -0.2012384152043576, + "PushCube-v1": 0.0742008167559442, + "StackCube-v1": 0.13235082030965997 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": -0.024000803025241515, + "low": -0.07546951652137965, + "mean": -0.05179500907541345, + "n": 144, + "num_samples": 300 + }, + "n": 144 + } + }, + "target": "utility_margin", + "tau": { + "LiftPegUpright-v1": 0.028235521236802175, + "PickCube-v1": 0.5025584143631503, + "PullCube-v1": -0.2012384152043573, + "PushCube-v1": 0.07420081675594419, + "StackCube-v1": 0.13235082030966006, + "__global__": 0.018528842023845943 + }, + "threshold_scope": "task", + "weights": [ + -0.06419078293938928, + 0.07864473400199715, + 0.05670874450563532, + 0.030682030137095147, + -0.017874347910056848, + -0.02068898371618812, + -0.14209815512665164, + -0.15088977540454895, + 0.058742278143481846, + 0.04013778871314058, + 0.029322121431418394, + 0.029091810230969208, + -0.04043658921086313, + -0.0310373634235519, + 0.00947510210158854, + -0.0061723529946101805, + -0.015421451059575333, + 0.03658962429356194, + -1.5425149460816038e-16, + -0.011433546961072683, + -0.09105471904483498, + 0.24636175694603346, + 0.2830253454973892, + 0.08308278332924052, + -0.18110157639978264, + -0.11400591941440252, + 0.006962277279499496, + -0.07089795409078059, + 0.23677529599504066, + 0.2830446516154337, + 5.828605755163547e-16, + 0.20725888095715322, + 0.21619497138712734, + 0.01867862192678118, + -0.09424784020821385, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] +} diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/metrics_by_seed.json b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..b3357bc80f2bef7327c10c6a793e72233115f334 --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/metrics_by_seed.json @@ -0,0 +1,3236 @@ +{ + "1073161778/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6218487394957983, + "pairwise_causal_calibration_confidence": 0.5252122988397412, + "pairwise_causal_calibration_ece": 0.09663644065605714, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.07106166187087547, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.67014479637146, + "selected_utility_gain_over_base": -1.32985520362854, + "selector_regret": 1.32985520362854, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.028235521236802175 + }, + "1073161778/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6083333333333333, + "pairwise_causal_calibration_confidence": 0.5228008383368301, + "pairwise_causal_calibration_ece": 0.08553249499650317, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.05584757187568534, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.028235521236802175 + }, + "1073161778/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5254991602652951, + "pairwise_causal_calibration_ece": 0.14116750640137155, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.07873777798305576, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.028235521236802175 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5381830912240237, + "pairwise_causal_calibration_ece": 0.10961166265259512, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1171582058773368, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "tau": 0.13235082030966006 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5386619397445, + "pairwise_causal_calibration_ece": 0.06848091739835714, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.11952381111596475, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "tau": 0.13235082030966006 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5353470658344741, + "pairwise_causal_calibration_ece": 0.036081505594097285, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.2072636199490521, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "tau": 0.13235082030966006 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5142857142857142, + "pairwise_causal_calibration_confidence": 0.5296341103177351, + "pairwise_causal_calibration_ece": 0.015348396032020872, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.12464332364425626, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "tau": 0.13235082030966006 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.44761904761904764, + "pairwise_causal_calibration_confidence": 0.528839557722319, + "pairwise_causal_calibration_ece": 0.08122051010327141, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.10435665617161227, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.38095238095238093, + "pairwise_causal_calibration_confidence": 0.530463392824977, + "pairwise_causal_calibration_ece": 0.14951101187259608, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.0965746708031926, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "tau": 0.13235082030966006 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.5174492210423474, + "pairwise_causal_calibration_ece": 0.06290376649689283, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": -0.08116638752440412, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5272727272727272, + "pairwise_causal_calibration_confidence": 0.5193562339693999, + "pairwise_causal_calibration_ece": 0.007916493303327332, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": -0.09547952648462638, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6190476190476191, + "pairwise_causal_calibration_confidence": 0.5192244871616464, + "pairwise_causal_calibration_ece": 0.0998231318859727, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.0006632054109868557, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5557404501598795, + "pairwise_causal_calibration_ece": 0.16288330730273656, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.09117531897647907, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "tau": 0.13235082030966006 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5544807305695781, + "pairwise_causal_calibration_ece": 0.10154169690861004, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.10108263691246488, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "tau": 0.13235082030966006 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5529109212469531, + "pairwise_causal_calibration_ece": 0.33862520696123877, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.05985917108029659, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "tau": 0.13235082030966006 + }, + "1324642411/train0": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5583333333333333, + "pairwise_causal_calibration_confidence": 0.5085746791327035, + "pairwise_causal_calibration_ece": 0.04975865420062986, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.5936226389105781, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11045536398887634, + "selected_utility_gain_over_base": 0.0039848387241363525, + "selector_regret": 0.03360468149185181, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "tau": 0.07420081675594419 + }, + "1324642411/train1": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.680672268907563, + "pairwise_causal_calibration_confidence": 0.5079157579147822, + "pairwise_causal_calibration_ece": 0.1727565109927809, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.3040303172517523, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11658626794815063, + "selected_utility_gain_over_base": 0.010115742683410645, + "selector_regret": 1.8834137320518494, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "1324642411/train2": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5181818181818182, + "pairwise_causal_calibration_confidence": 0.5077976736149897, + "pairwise_causal_calibration_ece": 0.010384144566828457, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.07036169250445845, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.89352947473526, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5289748634459925, + "pairwise_causal_calibration_ece": 0.1376918032206741, + "pairwise_causal_calibration_pairs": 75.0, + "predicted_margin": -0.29978209413001944, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5614035087719298, + "pairwise_causal_calibration_confidence": 0.5257629988292399, + "pairwise_causal_calibration_ece": 0.03564050994268986, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": -0.79693146060142, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8095238095238095, + "pairwise_causal_calibration_confidence": 0.5337083842579382, + "pairwise_causal_calibration_ece": 0.27581542526587133, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.6377766658396278, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.523230296021853, + "pairwise_causal_calibration_ece": 0.10656362935518632, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.08363360102690726, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.007176061626523733, + "selected_utility_gain_over_base": -0.09565818542614579, + "selector_regret": 0.008001843933016062, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "tau": -0.2012384152043573 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.575, + "pairwise_causal_calibration_confidence": 0.5259787015164451, + "pairwise_causal_calibration_ece": 0.049021298483554854, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.1560213376803143, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.030119141563773155, + "selected_utility_gain_over_base": -0.07271510548889637, + "selector_regret": 0.006891986355185509, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "tau": -0.2012384152043573 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5333333333333333, + "pairwise_causal_calibration_confidence": 0.5258876236158011, + "pairwise_causal_calibration_ece": 0.007445709717532223, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.046015598870960614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02716982364654541, + "selected_utility_gain_over_base": -0.07566442340612411, + "selector_regret": 0.031781893223524094, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "tau": -0.2012384152043573 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.30833333333333335, + "pairwise_causal_calibration_confidence": 0.5145099591738125, + "pairwise_causal_calibration_ece": 0.20617662584047913, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.09082588792911983, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.014822185039520264, + "selected_utility_gain_over_base": -0.07908087968826294, + "selector_regret": 0.04537618160247803, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "tau": -0.2012384152043573 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5141448492443679, + "pairwise_causal_calibration_ece": 0.09747818257770119, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.1449663100493438, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.006729066371917725, + "selected_utility_gain_over_base": -0.08717399835586548, + "selector_regret": 0.009855946525931358, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "tau": -0.2012384152043573 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.5185093923599849, + "pairwise_causal_calibration_ece": 0.08149060764001503, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.13654486820462297, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09997648000717163, + "selected_utility_gain_over_base": 0.006073415279388428, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "tau": -0.2012384152043573 + }, + "1480856810/train0": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8383838383838383, + "pairwise_causal_calibration_confidence": 0.5267941522147715, + "pairwise_causal_calibration_ece": 0.3115896861690668, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.8247860886762404, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "1480856810/train1": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5798319327731093, + "pairwise_causal_calibration_confidence": 0.5180113719626723, + "pairwise_causal_calibration_ece": 0.061820560810436964, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.5725138287112228, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21981161832809448, + "selected_utility_gain_over_base": 0.13277754187583923, + "selector_regret": 1.7801883816719055, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "1480856810/train2": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5630252100840336, + "pairwise_causal_calibration_confidence": 0.5179749462983955, + "pairwise_causal_calibration_ece": 0.045050263785638145, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.4285300550683504, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21894000470638275, + "selected_utility_gain_over_base": 0.1319059282541275, + "selector_regret": 1.7810599952936172, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.43333333333333335, + "pairwise_causal_calibration_confidence": 0.5388105224755593, + "pairwise_causal_calibration_ece": 0.10547718914222602, + "pairwise_causal_calibration_pairs": 90.0, + "predicted_margin": -0.11725750852840032, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.23809523809523808, + "pairwise_causal_calibration_confidence": 0.5371878138277117, + "pairwise_causal_calibration_ece": 0.29909257573247366, + "pairwise_causal_calibration_pairs": 63.0, + "predicted_margin": -0.10799597937369632, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4888888888888889, + "pairwise_causal_calibration_confidence": 0.5391151823956642, + "pairwise_causal_calibration_ece": 0.05022629350677536, + "pairwise_causal_calibration_pairs": 90.0, + "predicted_margin": -0.17204314625253667, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5358486568758676, + "pairwise_causal_calibration_ece": 0.03584865687586758, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": -0.37628277603589594, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "tau": 0.5025584143631503 + }, + "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2222222222222222, + "pairwise_causal_calibration_confidence": 0.5423563498452709, + "pairwise_causal_calibration_ece": 0.32013412762304877, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": -0.18697224010199912, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "tau": 0.5025584143631503 + }, + "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5411589331210354, + "pairwise_causal_calibration_ece": 0.12449226645436867, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": -0.2402513599376978, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "tau": 0.5025584143631503 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5117256505009358, + "pairwise_causal_calibration_ece": 0.15494101616573086, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.495453729546493, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.0347811970859766, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "tau": -0.2012384152043573 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.575, + "pairwise_causal_calibration_confidence": 0.5062538341615629, + "pairwise_causal_calibration_ece": 0.06874616583843707, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.18733344993267076, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0303483996540308, + "selected_utility_gain_over_base": -0.07416643761098385, + "selector_regret": 0.16857311688363552, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "tau": -0.2012384152043573 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.575, + "pairwise_causal_calibration_confidence": 0.5094928991923253, + "pairwise_causal_calibration_ece": 0.06550710080767463, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.3994404854817541, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04690708592534065, + "selected_utility_gain_over_base": -0.057607751339673996, + "selector_regret": 0.06619475409388542, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "tau": -0.2012384152043573 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7008547008547008, + "pairwise_causal_calibration_confidence": 0.5507814264578395, + "pairwise_causal_calibration_ece": 0.15007327439686133, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": 0.2626760081803321, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.17586350440979, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42727272727272725, + "pairwise_causal_calibration_confidence": 0.5448159860797186, + "pairwise_causal_calibration_ece": 0.11754325880699122, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.17364935131576656, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12732218205928802, + "selected_utility_gain_over_base": -0.6968143135309219, + "selector_regret": 1.872677817940712, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.36134453781512604, + "pairwise_causal_calibration_confidence": 0.5460279556731249, + "pairwise_causal_calibration_ece": 0.18468341785799877, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.15403622504955924, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15726424753665924, + "selected_utility_gain_over_base": -0.6668722480535507, + "selector_regret": 1.8427357524633408, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.04624259356089375, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.00797935515478132, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.06617060088497922, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "1790188787/train0": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.38181818181818183, + "pairwise_causal_calibration_confidence": 0.5077769577736889, + "pairwise_causal_calibration_ece": 0.12595877595550709, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.1484177313105067, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "1790188787/train1": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.37719298245614036, + "pairwise_causal_calibration_confidence": 0.5066944209005328, + "pairwise_causal_calibration_ece": 0.1295014384443925, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": 0.13132582655221522, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10585354268550873, + "selected_utility_gain_over_base": -0.01891392469406128, + "selector_regret": 1.8941464573144913, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "1790188787/train2": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.38181818181818183, + "pairwise_causal_calibration_confidence": 0.5083204445849703, + "pairwise_causal_calibration_ece": 0.12650226276678844, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.12795113455003831, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10904359817504883, + "selected_utility_gain_over_base": -0.01572386920452118, + "selector_regret": 1.8909564018249512, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.575, + "pairwise_causal_calibration_confidence": 0.549416493223813, + "pairwise_causal_calibration_ece": 0.08571587301053908, + "pairwise_causal_calibration_pairs": 40.0, + "predicted_margin": -0.0008074765493970848, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06247224286198616, + "selected_utility_gain_over_base": -0.009443063288927078, + "selector_regret": 1.9375277571380138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6530612244897959, + "pairwise_causal_calibration_confidence": 0.557604964366195, + "pairwise_causal_calibration_ece": 0.09545626012360084, + "pairwise_causal_calibration_pairs": 49.0, + "predicted_margin": -0.0650108915156051, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.045472364872694016, + "selected_utility_gain_over_base": -0.026442941278219223, + "selector_regret": 1.954527635127306, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.425, + "pairwise_causal_calibration_confidence": 0.5487264271441268, + "pairwise_causal_calibration_ece": 0.12372642714412677, + "pairwise_causal_calibration_pairs": 40.0, + "predicted_margin": -0.0006465763829108795, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08447941392660141, + "selected_utility_gain_over_base": 0.012564107775688171, + "selector_regret": 1.9155205860733986, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "1845489013/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32608695652173914, + "pairwise_causal_calibration_confidence": 0.5077729807205135, + "pairwise_causal_calibration_ece": 0.18168602419877433, + "pairwise_causal_calibration_pairs": 46.0, + "predicted_margin": -0.22001414669968217, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "1845489013/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 1.0, + "pairwise_causal_calibration_confidence": 0.5209647285753305, + "pairwise_causal_calibration_ece": 0.4790352714246695, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": -0.18822734864823312, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "1845489013/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.44, + "pairwise_causal_calibration_confidence": 0.5128055424573941, + "pairwise_causal_calibration_ece": 0.07280554245739407, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -0.1547908021155039, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "1999022574/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2571428571428571, + "pairwise_causal_calibration_confidence": 0.5358621206887306, + "pairwise_causal_calibration_ece": 0.2787192635458735, + "pairwise_causal_calibration_pairs": 70.0, + "predicted_margin": 0.0012762962711883319, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "1999022574/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5297587479178145, + "pairwise_causal_calibration_ece": 0.041669823510756876, + "pairwise_causal_calibration_pairs": 70.0, + "predicted_margin": -0.08747088123670077, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "1999022574/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42105263157894735, + "pairwise_causal_calibration_confidence": 0.5267734463002417, + "pairwise_causal_calibration_ece": 0.10572081472129435, + "pairwise_causal_calibration_pairs": 76.0, + "predicted_margin": -0.10186397965031571, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3949579831932773, + "pairwise_causal_calibration_confidence": 0.5282333871650522, + "pairwise_causal_calibration_ece": 0.1332754039717749, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.1490312481371141, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7026570439338684, + "selected_utility_gain_over_base": -0.020948290824890137, + "selector_regret": 1.2973429560661316, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.028235521236802175 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.575, + "pairwise_causal_calibration_confidence": 0.5289841828268461, + "pairwise_causal_calibration_ece": 0.04601581717315384, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.036105644427698036, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.028235521236802175 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6847826086956522, + "pairwise_causal_calibration_confidence": 0.531948622673026, + "pairwise_causal_calibration_ece": 0.1528339860226262, + "pairwise_causal_calibration_pairs": 92.0, + "predicted_margin": 0.09375170906247592, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6934840679168701, + "selected_utility_gain_over_base": -0.030121266841888428, + "selector_regret": 1.3065159320831299, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.028235521236802175 + }, + "2038594191/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.27586206896551724, + "pairwise_causal_calibration_confidence": 0.5740754232851301, + "pairwise_causal_calibration_ece": 0.2982133543196128, + "pairwise_causal_calibration_pairs": 29.0, + "predicted_margin": 0.19762473284895746, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04285776615142822, + "selected_utility_gain_over_base": -1.9571422338485718, + "selector_regret": 1.9571422338485718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "2038594191/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2962962962962963, + "pairwise_causal_calibration_confidence": 0.5667318952152414, + "pairwise_causal_calibration_ece": 0.27043559891894486, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 0.3209231912001711, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07674666494131088, + "selected_utility_gain_over_base": -1.9232533350586891, + "selector_regret": 1.9232533350586891, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "2038594191/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.5635629372227472, + "pairwise_causal_calibration_ece": 0.1090174826772927, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.31779873234459755, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.12977690994739532, + "selected_utility_gain_over_base": -1.8702230900526047, + "selector_regret": 1.8702230900526047, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.26666666666666666, + "pairwise_causal_calibration_confidence": 0.5290379603266726, + "pairwise_causal_calibration_ece": 0.13118564683000297, + "pairwise_causal_calibration_pairs": 60.0, + "predicted_margin": 0.15266150175970736, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.077283151447773, + "selected_candidate_index": 4.5, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0403175354003906, + "selected_utility_gain_over_base": -0.9596824645996094, + "selector_regret": 0.036965616047382355, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.922716848552227, + "tau": -0.2012384152043573 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6491228070175439, + "pairwise_causal_calibration_confidence": 0.5248824145830566, + "pairwise_causal_calibration_ece": 0.06212019621724363, + "pairwise_causal_calibration_pairs": 57.0, + "predicted_margin": 0.1715300859769803, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5798319327731093, + "pairwise_causal_calibration_confidence": 0.5308674006096046, + "pairwise_causal_calibration_ece": 0.024482266081752357, + "pairwise_causal_calibration_pairs": 59.5, + "predicted_margin": 0.16346432037867953, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5393032283831621, + "pairwise_causal_calibration_ece": 0.06094260950628667, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.12372212373443373, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "tau": 0.13235082030966006 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4722222222222222, + "pairwise_causal_calibration_confidence": 0.5362308522692728, + "pairwise_causal_calibration_ece": 0.06400863004705062, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.049366979703384384, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "tau": 0.13235082030966006 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5391540330154272, + "pairwise_causal_calibration_ece": 0.016401522540128385, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.015355053119340732, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "tau": 0.13235082030966006 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5882352941176471, + "pairwise_causal_calibration_confidence": 0.517501300906042, + "pairwise_causal_calibration_ece": 0.07073399321160512, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.04424689933210735, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8090915828943253, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6583333333333333, + "pairwise_causal_calibration_confidence": 0.5160911960797463, + "pairwise_causal_calibration_ece": 0.14224213725358703, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.13077847063672782, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02204875648021698, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "tau": 0.13235082030966006 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5666666666666667, + "pairwise_causal_calibration_confidence": 0.5172535124316141, + "pairwise_causal_calibration_ece": 0.04941315423505255, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.04386283650419778, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04297435283660889, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "tau": 0.13235082030966006 + }, + "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5731060606060606, + "pairwise_causal_calibration_confidence": 0.5325709273453999, + "pairwise_causal_calibration_ece": 0.12646220200877553, + "pairwise_causal_calibration_pairs": 109.5, + "predicted_margin": 0.2219557334710413, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0733563229441643, + "selected_candidate_index": 10.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.5, + "selected_utility": 1.0287093725055456, + "selected_utility_gain_over_base": 0.985974382609129, + "selector_regret": 0.04464695043861866, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04055231809616089, + "tau": -0.2012384152043573 + }, + "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5248188405797101, + "pairwise_causal_calibration_confidence": 0.5322397110343211, + "pairwise_causal_calibration_ece": 0.10215544676823522, + "pairwise_causal_calibration_pairs": 106.0, + "predicted_margin": 0.2843962133295508, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0376745946705341, + "selected_candidate_index": 8.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.5, + "selected_utility": 1.0095977680757642, + "selected_utility_gain_over_base": 0.9668627781793475, + "selector_regret": 0.028076826594769955, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.07623404636979103, + "tau": -0.2012384152043573 + }, + "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5214646464646465, + "pairwise_causal_calibration_confidence": 0.536003409987186, + "pairwise_causal_calibration_ece": 0.10720913222134038, + "pairwise_causal_calibration_pairs": 109.5, + "predicted_margin": 0.24145145365463994, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1359489113092422, + "selected_candidate_index": 14.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.5, + "selected_utility": 1.0227726306766272, + "selected_utility_gain_over_base": 0.9800376407802105, + "selector_regret": 0.11317628063261509, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5494505494505495, + "pairwise_causal_calibration_confidence": 0.5260813668824347, + "pairwise_causal_calibration_ece": 0.036558379935610945, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": 0.08783512114247624, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "selected_candidate_index": 11.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19761089980602264, + "selected_utility_gain_over_base": -0.4024796634912491, + "selector_regret": 0.01394665241241455, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "tau": 0.028235521236802175 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5218750448469349, + "pairwise_causal_calibration_ece": 0.049553526581636476, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": -0.13881394667704933, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "tau": 0.028235521236802175 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4945054945054945, + "pairwise_causal_calibration_confidence": 0.5232228710238422, + "pairwise_causal_calibration_ece": 0.028717376518347715, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": 0.14397102567158404, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22035211324691772, + "selected_utility_gain_over_base": -0.379738450050354, + "selector_regret": 0.009861335158348083, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "tau": 0.028235521236802175 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5444159724840614, + "pairwise_causal_calibration_ece": 0.04441597248406137, + "pairwise_causal_calibration_pairs": 60.0, + "predicted_margin": 0.0942295054093702, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7692307692307693, + "pairwise_causal_calibration_confidence": 0.5435121854300012, + "pairwise_causal_calibration_ece": 0.22571858380076806, + "pairwise_causal_calibration_pairs": 39.0, + "predicted_margin": 0.029952749201257664, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.38095238095238093, + "pairwise_causal_calibration_confidence": 0.5454433115811214, + "pairwise_causal_calibration_ece": 0.1644909306287403, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.004657759279381279, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "440577711/train0": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6206896551724138, + "pairwise_causal_calibration_confidence": 0.5273358125858131, + "pairwise_causal_calibration_ece": 0.09335384258660073, + "pairwise_causal_calibration_pairs": 29.0, + "predicted_margin": 0.5465320030087847, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "440577711/train1": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4, + "pairwise_causal_calibration_confidence": 0.5197720209730798, + "pairwise_causal_calibration_ece": 0.11977202097307982, + "pairwise_causal_calibration_pairs": 15.0, + "predicted_margin": 0.629918599045802, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "440577711/train2": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.46296296296296297, + "pairwise_causal_calibration_confidence": 0.5240762582462818, + "pairwise_causal_calibration_ece": 0.061113295283318825, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 0.6241310075694073, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.36363636363636365, + "pairwise_causal_calibration_confidence": 0.5267148410723766, + "pairwise_causal_calibration_ece": 0.163078477436013, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": 0.046645784157056724, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20773810148239136, + "selected_utility_gain_over_base": -0.4922170639038086, + "selector_regret": 1.7922618985176086, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.028235521236802175 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6909090909090909, + "pairwise_causal_calibration_confidence": 0.5234185829111347, + "pairwise_causal_calibration_ece": 0.16749050799795617, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": -0.17358234568930467, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "tau": 0.028235521236802175 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.5255994659745129, + "pairwise_causal_calibration_ece": 0.07105401142905837, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": -0.022380468606574055, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "tau": 0.028235521236802175 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6134453781512605, + "pairwise_causal_calibration_confidence": 0.5129229068912616, + "pairwise_causal_calibration_ece": 0.10052247125999891, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.4269940390592024, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11310504376888275, + "selected_utility_gain_over_base": 0.11310482025146484, + "selector_regret": 1.8868949562311172, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5619047619047619, + "pairwise_causal_calibration_confidence": 0.5125583130541626, + "pairwise_causal_calibration_ece": 0.049346448850599356, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.37833433181770104, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.07420081675594419 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5966386554621849, + "pairwise_causal_calibration_confidence": 0.5151111613972951, + "pairwise_causal_calibration_ece": 0.0815274940648898, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.282290647599035, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "tau": 0.07420081675594419 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5296207411344719, + "pairwise_causal_calibration_ece": 0.06533502684875758, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.10499618981526784, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09919407218694687, + "selected_utility_gain_over_base": -0.0825214758515358, + "selector_regret": 1.9008059278130531, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.5273196708342051, + "pairwise_causal_calibration_ece": 0.07268032916579492, + "pairwise_causal_calibration_pairs": 75.0, + "predicted_margin": 0.06533471439439086, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04792279005050659, + "selected_utility_gain_over_base": -0.13379275798797607, + "selector_regret": 1.9520772099494934, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7818181818181819, + "pairwise_causal_calibration_confidence": 0.529609307543077, + "pairwise_causal_calibration_ece": 0.25220887427510486, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.09915967492706912, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5619047619047619, + "pairwise_causal_calibration_confidence": 0.5561217648468477, + "pairwise_causal_calibration_ece": 0.03209834914225741, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.5011206164338899, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.274298369884491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.028235521236802175 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.49523809523809526, + "pairwise_causal_calibration_confidence": 0.5555371517280593, + "pairwise_causal_calibration_ece": 0.06029905648996403, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.2910589829678913, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.274298369884491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.028235521236802175 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5083333333333333, + "pairwise_causal_calibration_confidence": 0.5609759280259827, + "pairwise_causal_calibration_ece": 0.05264259469264974, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.4536901796255097, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6789124011993408, + "selected_utility_gain_over_base": -0.04678922891616821, + "selector_regret": 0.03281521797180176, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "tau": 0.028235521236802175 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6541666666666667, + "pairwise_causal_calibration_confidence": 0.5656883276900622, + "pairwise_causal_calibration_ece": 0.1169840028584393, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.5216712480085811, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04469547560438514, + "selected_candidate_index": 11.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.009900043485686183, + "selected_utility_gain_over_base": 0.004801700764801353, + "selector_regret": 0.034795432118698955, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12161866808310151, + "tau": -0.2012384152043573 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.375, + "pairwise_causal_calibration_confidence": 0.5625549213649634, + "pairwise_causal_calibration_ece": 0.19272229094655288, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.4816967914950128, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.035827874671667814, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.006450424436479807, + "selected_utility_gain_over_base": 0.0013520817155949771, + "selector_regret": 0.029377450235188007, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12975450651720166, + "tau": -0.2012384152043573 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6416666666666666, + "pairwise_causal_calibration_confidence": 0.5636531135559604, + "pairwise_causal_calibration_ece": 0.10846025594242703, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.39041494710515146, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04489925503730774, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01727360486984253, + "selected_utility_gain_over_base": 0.0121752621489577, + "selector_regret": 0.02762565016746521, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12068312615156174, + "tau": -0.2012384152043573 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5190311348285646, + "pairwise_causal_calibration_ece": 0.10236446816189787, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.4324165282764497, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.056092143058776855, + "selected_utility_gain_over_base": -0.027042903006076813, + "selector_regret": 0.0321691632270813, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "tau": -0.2012384152043573 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7, + "pairwise_causal_calibration_confidence": 0.5160986674159838, + "pairwise_causal_calibration_ece": 0.1839013325840162, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.0489088109005007, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.017789265140891075, + "selected_utility_gain_over_base": -0.0653457809239626, + "selector_regret": 0.04962191917002201, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "tau": -0.2012384152043573 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5129089770472632, + "pairwise_causal_calibration_ece": 0.012908977047263237, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.13337534819821673, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.015303333289921284, + "selected_utility_gain_over_base": -0.06783171277493238, + "selector_regret": 0.060348290018737316, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "tau": -0.2012384152043573 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5277777777777778, + "pairwise_causal_calibration_confidence": 0.5597351518736766, + "pairwise_causal_calibration_ece": 0.05616248877449882, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.07276876911827704, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "tau": 0.13235082030966006 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.562343415271088, + "pairwise_causal_calibration_ece": 0.11789897082664356, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.1554209538300596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11176733672618866, + "selected_utility_gain_over_base": -0.051239341497421265, + "selector_regret": 0.7011084705591202, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "tau": 0.13235082030966006 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5588372328026256, + "pairwise_causal_calibration_ece": 0.027075654701186606, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.11907170730895571, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "tau": 0.13235082030966006 + }, + "672328262/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8235294117647058, + "pairwise_causal_calibration_confidence": 0.5525242267379454, + "pairwise_causal_calibration_ece": 0.27100518502676063, + "pairwise_causal_calibration_pairs": 17.0, + "predicted_margin": -0.087075237393225, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "672328262/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.45714285714285713, + "pairwise_causal_calibration_confidence": 0.5406287558110948, + "pairwise_causal_calibration_ece": 0.08348589866823782, + "pairwise_causal_calibration_pairs": 35.0, + "predicted_margin": -0.23567605508015027, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "672328262/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4, + "pairwise_causal_calibration_confidence": 0.5442903896362881, + "pairwise_causal_calibration_ece": 0.1442903896362882, + "pairwise_causal_calibration_pairs": 30.0, + "predicted_margin": -0.14512362014342817, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04013325646519661, + "selected_utility_gain_over_base": -1.9598667435348034, + "selector_regret": 1.9598667435348034, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.2012384152043573 + }, + "701667486/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5238095238095238, + "pairwise_causal_calibration_confidence": 0.5307788011897017, + "pairwise_causal_calibration_ece": 0.00696927738017783, + "pairwise_causal_calibration_pairs": 42.0, + "predicted_margin": 0.18126309408722283, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "701667486/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5952380952380952, + "pairwise_causal_calibration_confidence": 0.5325935443428202, + "pairwise_causal_calibration_ece": 0.06264455089527499, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.15700516615012938, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "701667486/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5185185185185185, + "pairwise_causal_calibration_confidence": 0.5303820261796057, + "pairwise_causal_calibration_ece": 0.011863507661087236, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 0.1922641707596195, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.13235082030966006 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.49166666666666664, + "pairwise_causal_calibration_confidence": 0.5222884856095076, + "pairwise_causal_calibration_ece": 0.03062181894284094, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.08999271197759756, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.061248183250427246, + "selected_utility_gain_over_base": -1.9387518167495728, + "selector_regret": 0.04714745283126831, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "tau": -0.2012384152043573 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6833333333333333, + "pairwise_causal_calibration_confidence": 0.5252521749807111, + "pairwise_causal_calibration_ece": 0.15808115835262226, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.26165809689080527, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "tau": -0.2012384152043573 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.5187728818008999, + "pairwise_causal_calibration_ece": 0.08122711819910011, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.061796090652556146, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06438694149255753, + "selected_utility_gain_over_base": -1.9356130585074425, + "selector_regret": 0.06071493774652481, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "tau": -0.2012384152043573 + }, + "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.5043650793650793, + "pairwise_causal_calibration_confidence": 0.5207629106352885, + "pairwise_causal_calibration_ece": 0.08615947226034647, + "pairwise_causal_calibration_pairs": 108.0, + "predicted_margin": 0.005890085195897528, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 0.7526603279014429, + "selected_candidate_index": 0.6666666666666666, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.125017994393905, + "selected_utility_gain_over_base": 0.007424845049778621, + "selector_regret": 0.6276423335075378, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6553361676633358, + "tau": -0.20123841520435728 + }, + "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.533974358974359, + "pairwise_causal_calibration_confidence": 0.5196740117740427, + "pairwise_causal_calibration_ece": 0.13755775384340305, + "pairwise_causal_calibration_pairs": 101.66666666666667, + "predicted_margin": -0.19810309176890203, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3705032741030057, + "selected_candidate_index": 8.333333333333334, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.109618179500103, + "selected_utility_gain_over_base": -0.007974969844023386, + "selector_regret": 1.2608850946029027, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.03749322146177292, + "tau": -0.20123841520435728 + }, + "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.5404761904761904, + "pairwise_causal_calibration_confidence": 0.5222504044176294, + "pairwise_causal_calibration_ece": 0.10634938783489285, + "pairwise_causal_calibration_pairs": 108.0, + "predicted_margin": -0.15048924424583973, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3504486680030823, + "selected_candidate_index": 10.333333333333334, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09164678926269214, + "selected_utility_gain_over_base": -0.02594636008143425, + "selector_regret": 1.25880187874039, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.05754782756169637, + "tau": -0.20123841520435728 + }, + "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.75, + "execute_generated": 0.75, + "fallback_rate": 0.25, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "pairwise_causal_calibration_accuracy": 0.5482371794871794, + "pairwise_causal_calibration_confidence": 0.5286089316154619, + "pairwise_causal_calibration_ece": 0.1306918529485389, + "pairwise_causal_calibration_pairs": 119.25, + "predicted_margin": -0.028685105901352856, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0424291878007352, + "selected_candidate_index": 7.0, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5305264240596443, + "selected_utility_gain_over_base": -0.04267291794531047, + "selector_regret": 0.5119027637410909, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.06562284985557199, + "tau": -0.2012384152043573 + }, + "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.75, + "execute_generated": 0.75, + "fallback_rate": 0.25, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "pairwise_causal_calibration_accuracy": 0.5888404079580549, + "pairwise_causal_calibration_confidence": 0.5262713091494071, + "pairwise_causal_calibration_ece": 0.13822538375258459, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": -0.013317565754431063, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0629530912265182, + "selected_candidate_index": 7.0, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5966028785333037, + "selected_utility_gain_over_base": 0.023403536528348923, + "selector_regret": 0.4663502126932144, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.045098946429789066, + "tau": -0.2012384152043573 + }, + "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "pairwise_causal_calibration_accuracy": 0.5254749335631689, + "pairwise_causal_calibration_confidence": 0.5306118103750065, + "pairwise_causal_calibration_ece": 0.13666304064151388, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.02448240692729431, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0671670949086547, + "selected_candidate_index": 5.5, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5695318393409252, + "selected_utility_gain_over_base": -0.0036675026640295982, + "selector_regret": 0.4976352555677295, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.04088494274765253, + "tau": -0.2012384152043573 + } +} diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/metrics_by_task.json b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..3b3cccead676ca1682cc802928cca9b21a5d3d9c --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/metrics_by_task.json @@ -0,0 +1,137 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.2, + "base_utility": 0.9498705387115478, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.5333333333333333, + "pairwise_causal_calibration_accuracy": 0.5494360697557629, + "pairwise_causal_calibration_confidence": 0.5321483671612611, + "pairwise_causal_calibration_ece": 0.08779692862966537, + "pairwise_causal_calibration_pairs": 94.8, + "predicted_margin": 0.09934494198504117, + "proposal_oracle_success": 0.6, + "proposal_oracle_utility": 1.4122399787108104, + "selected_candidate_index": 7.2, + "selected_success": 0.26666666666666666, + "selected_success_gain_over_base": 0.06666666666666667, + "selected_utility": 0.9396337101856868, + "selected_utility_gain_over_base": -0.010236828525861105, + "selector_regret": 0.47343848447004955, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.4, + "support_gap": 0.5877600212891897, + "tau": 0.028235521236802175 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3796296296296296, + "pairwise_causal_calibration_confidence": 0.5397879799473912, + "pairwise_causal_calibration_ece": 0.16015835031776168, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": -0.2678354586918643, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13425676027933756, + "selected_candidate_index": 1.3333333333333333, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8657432397206624, + "tau": 0.5025584143631503 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.7751704612234608, + "coverage": 0.8133333333333334, + "execute_generated": 0.8133333333333334, + "fallback_rate": 0.18666666666666668, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.4666666666666667, + "pairwise_causal_calibration_accuracy": 0.5439315887395845, + "pairwise_causal_calibration_confidence": 0.5315713773974821, + "pairwise_causal_calibration_ece": 0.1202240575596694, + "pairwise_causal_calibration_pairs": 92.32, + "predicted_margin": 0.0587216273047277, + "proposal_oracle_success": 0.5333333333333333, + "proposal_oracle_utility": 1.1093241602430741, + "selected_candidate_index": 6.16, + "selected_success": 0.32, + "selected_success_gain_over_base": -0.04, + "selected_utility": 0.6822500004371007, + "selected_utility_gain_over_base": -0.09292046078636, + "selector_regret": 0.4528969661643108, + "success_selector_gap": 0.22666666666666666, + "success_support_gap": 0.02666666666666667, + "support_gap": 0.1075624164690574, + "tau": -0.2012384152043576 + }, + "PushCube-v1": { + "base_success": 0.16666666666666666, + "base_utility": 0.3924041787783305, + "coverage": 0.7777777777777778, + "execute_generated": 0.7777777777777778, + "fallback_rate": 0.2222222222222222, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.8333333333333334, + "pairwise_causal_calibration_accuracy": 0.5408758312375324, + "pairwise_causal_calibration_confidence": 0.5154259344288218, + "pairwise_causal_calibration_ece": 0.09497784791093052, + "pairwise_causal_calibration_pairs": 97.44444444444444, + "predicted_margin": 0.32846837901195985, + "proposal_oracle_success": 0.8888888888888888, + "proposal_oracle_utility": 1.7857814149724112, + "selected_candidate_index": 5.222222222222222, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.3333333333333333, + "selected_utility": 1.0611259018381436, + "selected_utility_gain_over_base": 0.6687217230598131, + "selector_regret": 0.7246555131342676, + "success_selector_gap": 0.3888888888888889, + "success_support_gap": 0.1111111111111111, + "support_gap": 0.21421858502758873, + "tau": 0.0742008167559442 + }, + "StackCube-v1": { + "base_success": 0.2727272727272727, + "base_utility": 0.7763947045261209, + "coverage": 0.21212121212121213, + "execute_generated": 0.21212121212121213, + "fallback_rate": 0.7878787878787878, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.6363636363636364, + "pairwise_causal_calibration_accuracy": 0.4871933998564077, + "pairwise_causal_calibration_confidence": 0.5389990977485094, + "pairwise_causal_calibration_ece": 0.10659066999978553, + "pairwise_causal_calibration_pairs": 67.45454545454545, + "predicted_margin": 0.03761618987646351, + "proposal_oracle_success": 0.5151515151515151, + "proposal_oracle_utility": 1.1567951707225856, + "selected_candidate_index": 5.606060606060606, + "selected_success": 0.30303030303030304, + "selected_success_gain_over_base": 0.030303030303030304, + "selected_utility": 0.769150389414845, + "selected_utility_gain_over_base": -0.007244315111275875, + "selector_regret": 0.3912377664537141, + "success_selector_gap": 0.21212121212121213, + "success_support_gap": 0.48484848484848486, + "support_gap": 0.8432048292774142, + "tau": 0.13235082030965997 + } +} diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/split_hash.txt b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..865ad41e869c1bfbc57e4db65ed79e91e6d93081 --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/split_hash.txt @@ -0,0 +1,14 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + } +} diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/table.tex b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..24b4bc35514f25fc28bd05c4dd73d3398434dc78 --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_learned_dominance_selector.py +\begin{tabular}{lrrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap & Cal. ECE \\ +\midrule +144 & 0.6389 & 0.3611 & 0.2917 & 0.3264 & 0.5694 & 0.5486 & 0.2014 & 0.2500 & 0.0221 \\ +\bottomrule +\end{tabular} diff --git a/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/train.log b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..34298d0999a299a6e4af72fc2eaa12f327267986 --- /dev/null +++ b/runs/ctt_base_context_obs_learned_dominance_score_chartcompat_utility_task_envclip_k16_train_to_test/train.log @@ -0,0 +1,5 @@ +trained ridge dominance calibrator on calibration measured rows only +fit_objective=pointwise +selected_lambda=100.0 +tau={"LiftPegUpright-v1": 0.028236, "PickCube-v1": 0.502558, "PullCube-v1": -0.201238, "PushCube-v1": 0.074201, "StackCube-v1": 0.132351, "__global__": 0.018529} +calibration_input=runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json